private int extractEvents(JobConfiguration jobConfiguration, JobTarget jobTarget, ControllerApi controllerApi, string eventType)
{
JArray listOfEvents = new JArray();
foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges)
{
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.To);
string eventsJSON = controllerApi.GetEvents(jobTarget.ApplicationID, eventType, fromTimeUnix, toTimeUnix);
if (eventsJSON != String.Empty)
{
// Load events
JArray eventsInHour = JArray.Parse(eventsJSON);
foreach (JObject interestingEvent in eventsInHour)
{
listOfEvents.Add(interestingEvent);
}
}
}
if (listOfEvents.Count > 0)
{
FileIOHelper.WriteJArrayToFile(listOfEvents, this.FilePathMap.EventsDataFilePath(jobTarget, eventType));
logger.Info("{0} {1} events from {2:o} to {3:o}", eventType, listOfEvents.Count, jobConfiguration.Input.TimeRange.From, jobConfiguration.Input.TimeRange.To);
loggerConsole.Info("{0} {1} events", eventType, listOfEvents.Count);
}
return(listOfEvents.Count);
}
internal HealthCheckRuleResult createHealthCheckRuleResult(
JobTarget jobTarget,
string entityType,
string entityName,
long entityID,
HealthCheckRuleDescription hcrd)
{
HealthCheckRuleResult healthCheckRuleResult = new HealthCheckRuleResult();
healthCheckRuleResult.Controller = jobTarget.Controller;
healthCheckRuleResult.Application = jobTarget.Application;
healthCheckRuleResult.ApplicationID = jobTarget.ApplicationID;
healthCheckRuleResult.EntityType = entityType;
healthCheckRuleResult.EntityName = entityName;
healthCheckRuleResult.EntityID = entityID;
healthCheckRuleResult.Category = hcrd.Category;
healthCheckRuleResult.Code = hcrd.Code;
healthCheckRuleResult.Name = hcrd.Name;
healthCheckRuleResult.Grade = 0;
healthCheckRuleResult.EvaluationTime = DateTime.Now;
return(healthCheckRuleResult);
}
private void readGranularRangeOfMetricsIntoEntities(
Dictionary <string, DBEntityBase> entitiesDictionaryByName,
JobTimeRange jobTimeRange,
JobTarget jobTarget,
List <MetricExtractMapping> entityMetricExtractMappingList,
string entityFolderName,
string entityType,
Dictionary <string, List <MetricValue> > metricValuesDictionary)
{
List <MetricExtractMapping> entityMetricExtractMappingListFiltered = entityMetricExtractMappingList.Where(m => m.EntityType == entityType).ToList();
foreach (MetricExtractMapping metricExtractMapping in entityMetricExtractMappingListFiltered)
{
List <AppDRESTMetric> metricData = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTMetric>(FilePathMap.MetricHourRangeDataFilePath(jobTarget, entityFolderName, metricExtractMapping.FolderName, jobTimeRange));
if (metricData != null)
{
List <MetricValue> metricValues = readMetricsIntoEntities(metricData, entitiesDictionaryByName, jobTarget, jobTimeRange);
if (metricValues != null)
{
if (metricValuesDictionary.ContainsKey(metricExtractMapping.FolderName) == false)
{
metricValuesDictionary.Add(metricExtractMapping.FolderName, metricValues);
}
else
{
metricValuesDictionary[metricExtractMapping.FolderName].AddRange(metricValues);
}
}
}
}
}
private int extractHealthRuleViolations(JobConfiguration jobConfiguration, JobTarget jobTarget, ControllerApi controllerApi)
{
JArray listOfHealthRuleViolations = new JArray();
foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges)
{
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.To);
string healthRuleViolationsJSON = controllerApi.GetHealthRuleViolations(jobTarget.ApplicationID, fromTimeUnix, toTimeUnix);
if (healthRuleViolationsJSON != String.Empty)
{
// Load health rule violations
JArray healthRuleViolationsInHour = JArray.Parse(healthRuleViolationsJSON);
foreach (JObject healthRuleViolation in healthRuleViolationsInHour)
{
listOfHealthRuleViolations.Add(healthRuleViolation);
}
}
}
if (listOfHealthRuleViolations.Count > 0)
{
FileIOHelper.WriteJArrayToFile(listOfHealthRuleViolations, FilePathMap.HealthRuleViolationsDataFilePath(jobTarget));
logger.Info("{0} health rule violations from {1:o} to {2:o}", listOfHealthRuleViolations.Count, jobConfiguration.Input.TimeRange.From, jobConfiguration.Input.TimeRange.To);
loggerConsole.Info("{0} health rule violations", listOfHealthRuleViolations.Count);
}
return(listOfHealthRuleViolations.Count);
}
private int extractAuditLogEvents(JobConfiguration jobConfiguration, JobTarget jobTarget, ControllerApi controllerApi)
{
JArray listOfAuditEvents = new JArray();
foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges)
{
string auditEventsJSON = controllerApi.GetAuditEvents(jobTimeRange.From, jobTimeRange.To);
if (auditEventsJSON != String.Empty)
{
JArray listOfAuditEventsInTimeRange = JArray.Parse(auditEventsJSON);
if (listOfAuditEventsInTimeRange != null)
{
// Load audit log events
foreach (JObject auditEvent in listOfAuditEventsInTimeRange)
{
listOfAuditEvents.Add(auditEvent);
}
}
}
}
if (listOfAuditEvents.Count > 0)
{
FileIOHelper.WriteJArrayToFile(listOfAuditEvents, FilePathMap.AuditEventsDataFilePath(jobTarget));
logger.Info("{0} audit events from {1:o} to {2:o}", listOfAuditEvents.Count, jobConfiguration.Input.TimeRange.From, jobConfiguration.Input.TimeRange.To);
loggerConsole.Info("{0} audit events", listOfAuditEvents.Count);
}
return(listOfAuditEvents.Count);
}
internal static JobTargetGroupData DeserializeJobTargetGroupData(JsonElement element)
{
ResourceIdentifier id = default;
string name = default;
ResourceType type = default;
SystemData systemData = default;
Optional <IList <JobTarget> > members = default;
foreach (var property in element.EnumerateObject())
{
if (property.NameEquals("id"))
{
id = new ResourceIdentifier(property.Value.GetString());
continue;
}
if (property.NameEquals("name"))
{
name = property.Value.GetString();
continue;
}
if (property.NameEquals("type"))
{
type = property.Value.GetString();
continue;
}
if (property.NameEquals("systemData"))
{
systemData = JsonSerializer.Deserialize <SystemData>(property.Value.ToString());
continue;
}
if (property.NameEquals("properties"))
{
if (property.Value.ValueKind == JsonValueKind.Null)
{
property.ThrowNonNullablePropertyIsNull();
continue;
}
foreach (var property0 in property.Value.EnumerateObject())
{
if (property0.NameEquals("members"))
{
if (property0.Value.ValueKind == JsonValueKind.Null)
{
property0.ThrowNonNullablePropertyIsNull();
continue;
}
List <JobTarget> array = new List <JobTarget>();
foreach (var item in property0.Value.EnumerateArray())
{
array.Add(JobTarget.DeserializeJobTarget(item));
}
members = array;
continue;
}
}
continue;
}
}
return(new JobTargetGroupData(id, name, type, systemData, Optional.ToList(members)));
}
private HealthCheckRuleResult evaluate_Controller_Setting_Performance_Profile(
HealthCheckRuleDescription hcrd,
JobTarget jobTarget,
Dictionary <string, HealthCheckSettingMapping> healthCheckSettingsDictionary,
List <ControllerSetting> controllerSettingsList)
{
logger.Trace("Evaluating {0}", hcrd);
HealthCheckRuleResult healthCheckRuleResult = createHealthCheckRuleResult(jobTarget, "Controller", jobTarget.Controller, 0, hcrd);
healthCheckRuleResult.Application = "[ALL APPS]";
if (controllerSettingsList != null)
{
ControllerSetting controllerSettingProfile = controllerSettingsList.Where(s => s.Name == "performance.profile").FirstOrDefault();
if (controllerSettingProfile == null)
{
healthCheckRuleResult.Grade = 1;
healthCheckRuleResult.Description = "Controller performance profile is unknown";
}
else
{
healthCheckRuleResult.Description = String.Format("Controller performance profile is '{0}'", controllerSettingProfile.Value);
// Defined in Enterprise Console via various
// C:\AppDynamics\Platform\platform-admin\archives\controller\4.5.4.15417\playbooks\controller-<size>.groovy
switch (controllerSettingProfile.Value.ToLower())
{
case "internal":
case "dev":
case "demo":
healthCheckRuleResult.Grade = 1;
break;
case "small":
healthCheckRuleResult.Grade = 2;
break;
case "medium":
healthCheckRuleResult.Grade = 3;
break;
case "large":
healthCheckRuleResult.Grade = 4;
break;
case "extra-large":
healthCheckRuleResult.Grade = 5;
break;
default:
healthCheckRuleResult.Grade = 1;
break;
}
}
}
return(healthCheckRuleResult);
}
/*
* 提供文本全局匹配“求职意向”，前缀使用精确匹配(意向行业)，段内可以使用模糊匹配（行业，例如51job的简历）
*/
public JobTarget extractJobTarget(string resumeStr)
{
JobTarget jobTarget = new JobTarget();
// resumeFullStr="期望月薪 6001-8000元/月 目前状况： 应届毕业生";
// resumeFullStr = "期望月薪 :面议6001-8000元/月 目前状况： 应届毕业生";
// 行业
string industry = getIndustry(resumeStr, true);
if (industry != null)
{
jobTarget.JobIndustry = industry;
}
// 职位
string jobCareer = getJobCareer(resumeStr, true);
if (jobCareer != null)
{
jobTarget.JobCareer = jobCareer;
}
// 薪资
string salary = getSalary(resumeStr, true);
if (salary != null)
{
jobTarget.Salary = salary;
}
// 地点
string location = getLocation(resumeStr, true);
if (location != null)
{
jobTarget.JobLocation = location;
}
// 性质
string jobCatagory = getJobCatagory(resumeStr, true);
if (jobCatagory != null)
{
jobTarget.JobCatagory = jobCatagory;
}
// 到岗时间
string enrollTime = getEnrollTime(resumeStr, true);
if (enrollTime != null)
{
jobTarget.EnrollTime = enrollTime;
}
return(jobTarget);
}
/*
* 提供段内“求职意向”按行匹配结果
*/
public JobTarget extractJobTarget(int start, int end)
{
JobTarget jobTarget = new JobTarget();
//获取段内所有数据
string line = getContent(start, end);
// 行业
string industry = getIndustry(line, false);
if (industry != null)
{
jobTarget.JobIndustry = industry;
}
// 职位
string jobCareer = getJobCareer(line, false);
if (jobCareer != null)
{
jobTarget.JobCareer = jobCareer;
}
// 薪资
string salary = getSalary(line, false);
if (salary != null)
{
jobTarget.Salary = salary;
}
// 地点
string location = getLocation(line, false);
if (location != null)
{
jobTarget.JobLocation = location;
}
// 性质
string jobCatagory = getJobCatagory(line, false);
if (jobCatagory != null)
{
jobTarget.JobCatagory = jobCatagory;
}
// 到岗时间
string enrollTime = getEnrollTime(line, false);
if (enrollTime != null)
{
jobTarget.EnrollTime = enrollTime;
}
return(jobTarget);
}
public JobTarget searchJobTarget()
{
// 从每个段中去搜索求职意向
JobTarget jobTarget = null;
StringBuilder resumeTxtBuilder = new StringBuilder();
foreach (string line in resumeContentList)
{
resumeTxtBuilder.Append(line + " ");
}
jobTarget = extractJobTarget(resumeTxtBuilder.ToString());
return(jobTarget);
}
/// <summary>
/// Version of Controller should be reasonably latest
/// </summary>
/// <param name="jobTarget"></param>
/// <param name="healthCheckSettingsDictionary"></param>
/// <param name="controllerSummariesList"></param>
/// <returns></returns>
private HealthCheckRuleResult evaluate_Controller_Version(
HealthCheckRuleDescription hcrd,
JobTarget jobTarget,
Dictionary <string, HealthCheckSettingMapping> healthCheckSettingsDictionary,
List <ControllerSummary> controllerSummariesList)
{
logger.Trace("Evaluating {0}", hcrd);
HealthCheckRuleResult healthCheckRuleResult = createHealthCheckRuleResult(jobTarget, "Controller", jobTarget.Controller, 0, hcrd);
healthCheckRuleResult.Application = "[ALL APPS]";
if (controllerSummariesList != null && controllerSummariesList.Count > 0)
{
ControllerSummary controllerSummary = controllerSummariesList[0];
Version versionThisController = new Version(jobTarget.ControllerVersion);
healthCheckRuleResult.Description = String.Format("The Controller version '{0}'", jobTarget.ControllerVersion, getVersionSetting(healthCheckSettingsDictionary, "ControllerVersionGrade5", "4.5"));
if (versionThisController >= getVersionSetting(healthCheckSettingsDictionary, "ControllerVersionGrade5", "4.5"))
{
healthCheckRuleResult.Grade = 5;
}
else if (versionThisController >= getVersionSetting(healthCheckSettingsDictionary, "ControllerVersionGrade4", "4.4"))
{
healthCheckRuleResult.Grade = 4;
}
else if (versionThisController >= getVersionSetting(healthCheckSettingsDictionary, "ControllerVersionGrade3", "4.3"))
{
healthCheckRuleResult.Grade = 3;
}
else if (versionThisController >= getVersionSetting(healthCheckSettingsDictionary, "ControllerVersionGrade2", "4.2"))
{
healthCheckRuleResult.Grade = 2;
}
else
{
healthCheckRuleResult.Grade = 1;
}
}
else
{
healthCheckRuleResult.Grade = 1;
healthCheckRuleResult.Description = "No information about Controller version available";
}
return(healthCheckRuleResult);
}
private int extractEvents(JobConfiguration jobConfiguration, JobTarget jobTarget, ControllerApi controllerApi, string eventType)
{
loggerConsole.Info("Extract List of Events {0} ({1} time ranges)", eventType, jobConfiguration.Input.HourlyTimeRanges.Count);
JArray listOfEventsArray = new JArray();
if (File.Exists(FilePathMap.ApplicationEventsDataFilePath(jobTarget, eventType)) == false)
{
foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges)
{
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.To);
try
{
string eventsJSON = controllerApi.GetApplicationEvents(jobTarget.ApplicationID, eventType, fromTimeUnix, toTimeUnix);
if (eventsJSON != String.Empty)
{
// Load events
JArray eventsInHourArray = JArray.Parse(eventsJSON);
foreach (JObject eventObject in eventsInHourArray)
{
listOfEventsArray.Add(eventObject);
}
Console.Write("[{0}]+", eventsInHourArray.Count);
}
}
catch (Exception ex)
{
logger.Warn(ex);
logger.Warn("Unable to parse JSON for Events Application={0}, EventType={1}, From={2}, To={3}", jobTarget.ApplicationID, eventType, fromTimeUnix, toTimeUnix);
}
}
if (listOfEventsArray.Count > 0)
{
FileIOHelper.WriteJArrayToFile(listOfEventsArray, FilePathMap.ApplicationEventsDataFilePath(jobTarget, eventType));
logger.Info("{0} {1} events from {2:o} to {3:o}", eventType, listOfEventsArray.Count, jobConfiguration.Input.TimeRange.From, jobConfiguration.Input.TimeRange.To);
loggerConsole.Info("{0} {1} events", eventType, listOfEventsArray.Count);
}
}
return(listOfEventsArray.Count);
}
public void extractJobTarget()
{
string seg_name = "career_objective";
if (segmentMap.ContainsKey(seg_name))
{
TargetJobAnalyzer analyzer = new TargetJobAnalyzer(resumeContentList);
SectionInfo sectionInfo = segmentMap[seg_name];
JobTarget jobTarget = analyzer.extractJobTarget(sectionInfo.Start, sectionInfo.End);
resumedata.JobTarget = jobTarget;
}
else
{
TargetJobAnalyzer analyzer = new TargetJobAnalyzer(resumeContentList);
JobTarget jobTarget = analyzer.searchJobTarget();
resumedata.JobTarget = jobTarget;
}
}
private int extractHealthRuleViolations(JobConfiguration jobConfiguration, JobTarget jobTarget, ControllerApi controllerApi)
{
JArray listOfHealthRuleViolations = new JArray();
if (File.Exists(FilePathMap.ApplicationHealthRuleViolationsDataFilePath(jobTarget)) == false)
{
foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges)
{
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.To);
string healthRuleViolationsJSON = controllerApi.GetApplicationHealthRuleViolations(jobTarget.ApplicationID, fromTimeUnix, toTimeUnix);
if (healthRuleViolationsJSON != String.Empty)
{
try
{
// Load health rule violations
JArray healthRuleViolationsInHourArray = JArray.Parse(healthRuleViolationsJSON);
foreach (JObject healthRuleViolationObject in healthRuleViolationsInHourArray)
{
listOfHealthRuleViolations.Add(healthRuleViolationObject);
}
}
catch (Exception)
{
logger.Warn("Unable to parse JSON for HR Violations Application={0}, From={1}, To={2}", jobTarget.ApplicationID, fromTimeUnix, toTimeUnix);
}
}
}
if (listOfHealthRuleViolations.Count > 0)
{
FileIOHelper.WriteJArrayToFile(listOfHealthRuleViolations, FilePathMap.ApplicationHealthRuleViolationsDataFilePath(jobTarget));
logger.Info("{0} health rule violations from {1:o} to {2:o}", listOfHealthRuleViolations.Count, jobConfiguration.Input.TimeRange.From, jobConfiguration.Input.TimeRange.To);
loggerConsole.Info("{0} health rule violations", listOfHealthRuleViolations.Count);
}
}
return(listOfHealthRuleViolations.Count);
}
private void extractListOfMetrics(
JobTarget jobTarget,
JobConfiguration jobConfiguration,
string metricPathPrefix,
string fileNamePrefix,
int maxDepth)
{
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
DateTime startTime = jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1].From;
DateTime endTime = jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1].To;
StringBuilder sbMetricPath = new StringBuilder(128);
sbMetricPath.Append(metricPathPrefix);
for (int currentDepth = 0; currentDepth <= maxDepth; currentDepth++)
{
sbMetricPath.Append("|*");
string metricPath = sbMetricPath.ToString();
loggerConsole.Trace("Depth {0:00}/{1:00}, {2}", currentDepth, maxDepth, metricPath);
logger.Info("Retrieving metric lists for Application {0}({1}), Metric='{2}', From {3:o}, To {4:o}", jobTarget.Application, jobTarget.ApplicationID, metricPath, startTime, endTime);
string metricsJson = String.Empty;
string metricsDataFilePath = FilePathMap.MetricsListDataFilePath(jobTarget, fileNamePrefix, currentDepth);
if (File.Exists(metricsDataFilePath) == false)
{
metricsJson = controllerApi.GetMetricData(
jobTarget.ApplicationID,
metricPath,
UnixTimeHelper.ConvertToUnixTimestamp(startTime),
UnixTimeHelper.ConvertToUnixTimestamp(endTime),
true);
FileIOHelper.SaveFileToPath(metricsJson, metricsDataFilePath);
}
}
}
}
/// <summary>
/// Converts a Job Target model to an AzureSqlElasticJobTargetModel
/// </summary>
/// <param name="resourceGroupName">The resource group name</param>
/// <param name="serverName">The server name</param>
/// <param name="agentName">The agent name</param>
/// <param name="targetGroupName">The target group name</param>
/// <param name="target">The JobTarget model</param>
/// <returns></returns>
protected AzureSqlElasticJobTargetModel CreateAzureSqlElasticJobTargetModel(
string resourceGroupName,
string serverName,
string agentName,
string targetGroupName,
JobTarget target)
{
return(new AzureSqlElasticJobTargetModel
{
ResourceGroupName = resourceGroupName,
ServerName = serverName,
AgentName = agentName,
TargetGroupName = targetGroupName,
MembershipType = target.MembershipType,
TargetType = target.Type,
RefreshCredentialName = new ResourceIdentifier(target.RefreshCredential).ResourceName,
TargetServerName = target.ServerName,
TargetDatabaseName = target.DatabaseName,
TargetElasticPoolName = target.ElasticPoolName,
TargetShardMapName = target.ShardMapName,
});
}
/// <summary>
/// We like SaaS controllers more than on premises
/// </summary>
/// <param name="jobTarget"></param>
/// <param name="healthCheckSettingsDictionary"></param>
/// <returns></returns>
private HealthCheckRuleResult evaluate_Controller_SaaS_OnPrem(
HealthCheckRuleDescription hcrd,
JobTarget jobTarget,
Dictionary <string, HealthCheckSettingMapping> healthCheckSettingsDictionary)
{
logger.Trace("Evaluating {0}", hcrd);
HealthCheckRuleResult healthCheckRuleResult = createHealthCheckRuleResult(jobTarget, "Controller", jobTarget.Controller, 0, hcrd);
healthCheckRuleResult.Application = "[ALL APPS]";
if (jobTarget.Controller.ToLower().Contains("saas.appdynamics") == true)
{
healthCheckRuleResult.Grade = 5;
healthCheckRuleResult.Description = "Controller is running in AppDynamics SaaS cloud";
}
else
{
healthCheckRuleResult.Grade = 3;
healthCheckRuleResult.Description = "Controller is running in OnPremises configuration";
}
return(healthCheckRuleResult);
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_SIM) == 0)
{
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_SIM)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
#region Prepare time range
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To);
long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000);
#endregion
#region Tiers
loggerConsole.Info("List of Tiers");
string tiersJSON = controllerApi.GetSIMListOfTiers();
if (tiersJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(tiersJSON, FilePathMap.SIMTiersDataFilePath(jobTarget));
}
#endregion
#region Nodes
loggerConsole.Info("List of Nodes");
string nodesJSON = controllerApi.GetSIMListOfNodes();
if (nodesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodesJSON, FilePathMap.SIMNodesDataFilePath(jobTarget));
}
#endregion
#region Groups
loggerConsole.Info("List of Groups");
string groupsJSON = controllerApi.GetSIMListOfGroups();
if (groupsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(groupsJSON, FilePathMap.SIMGroupsDataFilePath(jobTarget));
}
#endregion
#region Service Availability
loggerConsole.Info("List of Service Availability");
string sasJSON = controllerApi.GetSIMListOfServiceAvailability();
if (sasJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(sasJSON, FilePathMap.SIMServiceAvailabilitiesDataFilePath(jobTarget));
}
JArray sasList = FileIOHelper.LoadJArrayFromFile(FilePathMap.SIMServiceAvailabilitiesDataFilePath(jobTarget));
if (sasList != null)
{
loggerConsole.Info("Service Availability Details ({0} entities)", sasList.Count);
foreach (JToken saToken in sasList)
{
string saJSON = controllerApi.GetSIMServiceAvailability((long)saToken["id"]);
if (saJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(saJSON, FilePathMap.SIMServiceAvailabilityDataFilePath(jobTarget, saToken["name"].ToString(), (long)saToken["id"]));
}
string saEventsJSON = controllerApi.GetSIMServiceAvailabilityEvents((long)saToken["id"], fromTimeUnix, toTimeUnix, differenceInMinutes);
if (saEventsJSON != String.Empty && saEventsJSON != "[]")
{
FileIOHelper.SaveFileToPath(saEventsJSON, FilePathMap.SIMServiceAvailabilityEventsDataFilePath(jobTarget, saToken["name"].ToString(), (long)saToken["id"], jobConfiguration.Input.TimeRange));
}
}
loggerConsole.Info("Completed {0} Service Availabilities", sasList.Count);
}
#endregion
#region Machines
loggerConsole.Info("List of Machines");
string machinesJSON = controllerApi.GetSIMListOfMachines();
if (machinesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(machinesJSON, FilePathMap.SIMMachinesDataFilePath(jobTarget));
}
JArray machinesArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.SIMMachinesDataFilePath(jobTarget));
if (machinesArray != null)
{
loggerConsole.Info("Machine, Container Details and Processes ({0} entities)", machinesArray.Count);
int j = 0;
var listOfMachinesChunks = machinesArray.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_MACHINES_TO_PROCESS_PER_THREAD);
Parallel.ForEach <List <JToken>, int>(
listOfMachinesChunks,
new ParallelOptions {
MaxDegreeOfParallelism = MACHINES_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(listOfMachinesChunk, loop, subtotal) =>
{
// Set up controller access
using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
foreach (JToken machineToken in listOfMachinesChunk)
{
if (File.Exists(FilePathMap.SIMMachineDataFilePath(jobTarget, machineToken["name"].ToString(), (long)machineToken["id"])) == false)
{
string machineJSON = controllerApi.GetSIMMachine((long)machineToken["id"]);
if (machineJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(machineJSON, FilePathMap.SIMMachineDataFilePath(jobTarget, machineToken["name"].ToString(), (long)machineToken["id"]));
}
string machineDockerJSON = controllerApi.GetSIMMachineDockerContainers((long)machineToken["id"]);
if (machineDockerJSON != String.Empty && machineDockerJSON != "[]")
{
FileIOHelper.SaveFileToPath(machineDockerJSON, FilePathMap.SIMMachineDockerContainersDataFilePath(jobTarget, machineToken["name"].ToString(), (long)machineToken["id"]));
}
string machineProcessesJSON = controllerApi.GetSIMMachineProcesses((long)machineToken["id"], fromTimeUnix, toTimeUnix, differenceInMinutes);
if (machineProcessesJSON != String.Empty && machineProcessesJSON != "[]")
{
FileIOHelper.SaveFileToPath(machineProcessesJSON, FilePathMap.SIMMachineProcessesDataFilePath(jobTarget, machineToken["name"].ToString(), (long)machineToken["id"], jobConfiguration.Input.TimeRange));
}
}
}
return(listOfMachinesChunk.Count);
}
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
Console.Write("[{0}].", j);
}
);
loggerConsole.Info("Completed {0} Machines", machinesArray.Count);
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
bool haveProcessedAtLeastOneDBCollector = false;
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
if (jobTarget.Type == APPLICATION_TYPE_DB)
{
if (haveProcessedAtLeastOneDBCollector == false)
{
haveProcessedAtLeastOneDBCollector = true;
}
else
{
continue;
}
}
int numEventsTotal = 0;
#region Health Rule violations
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
loggerConsole.Info("Extract List of Health Rule Violations ({0} time ranges)", jobConfiguration.Input.HourlyTimeRanges.Count);
numEventsTotal = numEventsTotal + extractHealthRuleViolations(jobConfiguration, jobTarget, controllerApi);
}
#endregion
#region Events
loggerConsole.Info("Extract {0} event types ({1} time ranges)", EVENT_TYPES.Count, jobConfiguration.Input.HourlyTimeRanges.Count);
Parallel.ForEach(
EVENT_TYPES,
new ParallelOptions {
MaxDegreeOfParallelism = EVENTS_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(eventType, loop, subtotal) =>
{
using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
int numEventsInType = extractEvents(jobConfiguration, jobTarget, controllerApiParallel, eventType);
subtotal = subtotal + numEventsInType;
return(subtotal);
}
},
(finalResult) =>
{
Interlocked.Add(ref numEventsTotal, finalResult);
}
);
loggerConsole.Info("{0} events total", numEventsTotal);
#endregion
stepTimingTarget.NumEntities = numEventsTotal;
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(programOptions, jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_APM) == 0)
{
logger.Warn("No {0} targets to process", APPLICATION_TYPE_APM);
loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_APM);
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_APM)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 9;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region Prepare time range
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1].From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1].To);
long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000);
#endregion
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
#region Application
loggerConsole.Info("Application Name");
string applicationJSON = controllerApi.GetAPMApplication(jobTarget.ApplicationID);
if (applicationJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationJSON, FilePathMap.APMApplicationDataFilePath(jobTarget));
}
#endregion
#region Tiers
loggerConsole.Info("List of Tiers");
string tiersJSON = controllerApi.GetAPMTiers(jobTarget.ApplicationID);
if (tiersJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(tiersJSON, FilePathMap.APMTiersDataFilePath(jobTarget));
}
#endregion
#region Nodes
loggerConsole.Info("List of Nodes");
string nodesJSON = controllerApi.GetAPMNodes(jobTarget.ApplicationID);
if (nodesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodesJSON, FilePathMap.APMNodesDataFilePath(jobTarget));
}
#endregion
#region Backends
loggerConsole.Info("List of Backends");
string backendsJSON = controllerApi.GetAPMBackends(jobTarget.ApplicationID);
if (backendsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(backendsJSON, FilePathMap.APMBackendsDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
backendsJSON = controllerApi.GetAPMBackendsAdditionalDetail(jobTarget.ApplicationID);
if (backendsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(backendsJSON, FilePathMap.APMBackendsDetailDataFilePath(jobTarget));
}
List <AppDRESTBackend> backendsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBackend>(FilePathMap.APMBackendsDataFilePath(jobTarget));
if (backendsList != null)
{
loggerConsole.Info("DBMon Mappings for Backends ({0} entities)", backendsList.Count);
int j = 0;
var listOfBackendsInHourChunks = backendsList.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_BACKENDS_TO_PROCESS_PER_THREAD);
ParallelOptions parallelOptions = new ParallelOptions();
if (programOptions.ProcessSequentially == true)
{
parallelOptions.MaxDegreeOfParallelism = 1;
}
else
{
parallelOptions.MaxDegreeOfParallelism = BACKEND_PROPERTIES_EXTRACT_NUMBER_OF_THREADS;
}
Parallel.ForEach <List <AppDRESTBackend>, int>(
listOfBackendsInHourChunks,
parallelOptions,
() => 0,
(listOfBackendsInHourChunk, loop, subtotal) =>
{
// Set up controller access
ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
// Login into private API
controllerApiParallel.PrivateApiLogin();
foreach (AppDRESTBackend backend in listOfBackendsInHourChunk)
{
if (File.Exists(FilePathMap.APMBackendToDBMonMappingDataFilePath(jobTarget, backend)) == false)
{
string backendToDBMonMappingJSON = controllerApi.GetAPMBackendToDBMonMapping(backend.id);
if (backendToDBMonMappingJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(backendToDBMonMappingJSON, FilePathMap.APMBackendToDBMonMappingDataFilePath(jobTarget, backend));
}
}
}
return(listOfBackendsInHourChunk.Count);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
Console.Write("[{0}].", j);
}
);
loggerConsole.Info("Completed {0} Backends", backendsList.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + backendsList.Count;
}
#endregion
#region Business Transactions
loggerConsole.Info("List of Business Transactions");
string businessTransactionsJSON = controllerApi.GetAPMBusinessTransactions(jobTarget.ApplicationID);
if (businessTransactionsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(businessTransactionsJSON, FilePathMap.APMBusinessTransactionsDataFilePath(jobTarget));
}
#endregion
#region Service Endpoints
loggerConsole.Info("List of Service Endpoints");
string serviceEndPointsJSON = controllerApi.GetAPMServiceEndpoints(jobTarget.ApplicationID);
if (serviceEndPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(serviceEndPointsJSON, FilePathMap.APMServiceEndpointsDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
serviceEndPointsJSON = controllerApi.GetAPMServiceEndpointsAdditionalDetail(jobTarget.ApplicationID);
if (serviceEndPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(serviceEndPointsJSON, FilePathMap.APMServiceEndpointsDetailDataFilePath(jobTarget));
}
#endregion
#region Errors
loggerConsole.Info("List of Errors");
string errorsJSON = controllerApi.GetAPMErrors(jobTarget.ApplicationID);
if (errorsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(errorsJSON, FilePathMap.APMErrorsDataFilePath(jobTarget));
}
#endregion
#region Information Points
loggerConsole.Info("List of Information Points");
string informationPointsJSON = controllerApi.GetAPMInformationPoints(jobTarget.ApplicationID);
if (informationPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(informationPointsJSON, FilePathMap.APMInformationPointsDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
string informationPointsDetailJSON = controllerApi.GetAPMInformationPointsAdditionalDetail(jobTarget.ApplicationID);
if (informationPointsDetailJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(informationPointsDetailJSON, FilePathMap.APMInformationPointsDetailDataFilePath(jobTarget));
}
#endregion
#region Node Properties
List <AppDRESTNode> nodesList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTNode>(FilePathMap.APMNodesDataFilePath(jobTarget));
if (nodesList != null)
{
loggerConsole.Info("Node Properties for Nodes ({0} entities)", nodesList.Count);
int j = 0;
var listOfNodesInHourChunks = nodesList.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_NODES_TO_PROCESS_PER_THREAD);
ParallelOptions parallelOptions = new ParallelOptions();
if (programOptions.ProcessSequentially == true)
{
parallelOptions.MaxDegreeOfParallelism = 1;
}
else
{
parallelOptions.MaxDegreeOfParallelism = NODE_PROPERTIES_EXTRACT_NUMBER_OF_THREADS;
}
Parallel.ForEach <List <AppDRESTNode>, int>(
listOfNodesInHourChunks,
parallelOptions,
() => 0,
(listOfNodesInHourChunk, loop, subtotal) =>
{
// Set up controller access
ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
// Login into private API
controllerApiParallel.PrivateApiLogin();
foreach (AppDRESTNode node in listOfNodesInHourChunk)
{
if (File.Exists(FilePathMap.APMNodeRuntimePropertiesDataFilePath(jobTarget, node)) == false)
{
string nodePropertiesJSON = controllerApi.GetAPMNodeProperties(node.id);
if (nodePropertiesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodePropertiesJSON, FilePathMap.APMNodeRuntimePropertiesDataFilePath(jobTarget, node));
}
}
if (File.Exists(FilePathMap.APMNodeMetadataDataFilePath(jobTarget, node)) == false)
{
string nodeMetadataJSON = controllerApi.GetAPMNodeMetadata(jobTarget.ApplicationID, node.id);
if (nodeMetadataJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodeMetadataJSON, FilePathMap.APMNodeMetadataDataFilePath(jobTarget, node));
}
}
}
return(listOfNodesInHourChunk.Count);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
Console.Write("[{0}].", j);
}
);
loggerConsole.Info("Completed {0} Nodes", nodesList.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + nodesList.Count;
}
#endregion
#region Backend to Tier Mappings
List <AppDRESTTier> tiersRESTList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTTier>(FilePathMap.APMTiersDataFilePath(jobTarget));
if (tiersRESTList != null)
{
loggerConsole.Info("Backend to Tier Mappings ({0} entities)", tiersRESTList.Count);
int j = 0;
foreach (AppDRESTTier tier in tiersRESTList)
{
string backendMappingsJSON = controllerApi.GetAPMBackendToTierMapping(tier.id);
if (backendMappingsJSON != String.Empty && backendMappingsJSON != "[ ]")
{
FileIOHelper.SaveFileToPath(backendMappingsJSON, FilePathMap.APMBackendToTierMappingDataFilePath(jobTarget, tier));
}
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
j++;
}
loggerConsole.Info("Completed {0} Tiers", tiersRESTList.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + tiersRESTList.Count;
}
#endregion
#region Overflow Business Transactions
if (tiersRESTList != null)
{
loggerConsole.Info("Contents of Overflow Business Transaction in Tiers ({0} entities)", tiersRESTList.Count);
int j = 0;
foreach (AppDRESTTier tier in tiersRESTList)
{
JArray droppedBTsArray = new JArray();
JArray droppedBTsDebugModeArray = new JArray();
bool noMoreBTs = false;
long currentFetchedEventCount = 0;
long endEventID = 0;
while (noMoreBTs == false)
{
string batchOfBTsJSON = controllerApi.GetAPMBusinessTransactionsInOverflow(tier.id, currentFetchedEventCount, endEventID, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (batchOfBTsJSON != String.Empty)
{
JObject batchOfBTsContainer = JObject.Parse(batchOfBTsJSON);
if (batchOfBTsContainer != null)
{
// Copy out both of the containers, not sure why there are multiple
if (isTokenPropertyNull(batchOfBTsContainer, "droppedTransactionItemList") == false)
{
foreach (JObject btObject in batchOfBTsContainer["droppedTransactionItemList"])
{
droppedBTsArray.Add(btObject);
}
}
if (isTokenPropertyNull(batchOfBTsContainer, "debugModeDroppedTransactionItemList") == false)
{
foreach (JObject btObject in batchOfBTsContainer["debugModeDroppedTransactionItemList"])
{
droppedBTsDebugModeArray.Add(btObject);
}
}
currentFetchedEventCount = getLongValueFromJToken(batchOfBTsContainer, "eventSummariesCount");
endEventID = getLongValueFromJToken(batchOfBTsContainer, "endEventId");
if (currentFetchedEventCount == 0 || endEventID == 0)
{
// Done getting batches
noMoreBTs = true;
}
}
}
else
{
noMoreBTs = true;
}
}
if (droppedBTsArray.Count > 0)
{
FileIOHelper.SaveFileToPath(droppedBTsArray.ToString(), FilePathMap.APMTierOverflowBusinessTransactionRegularDataFilePath(jobTarget, tier));
}
if (droppedBTsDebugModeArray.Count > 0)
{
FileIOHelper.SaveFileToPath(droppedBTsDebugModeArray.ToString(), FilePathMap.APMTierOverflowBusinessTransactionDebugDataFilePath(jobTarget, tier));
}
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
j++;
}
loggerConsole.Info("Completed {0} Tiers", tiersRESTList.Count);
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public void DisplayJobTargetEndedStatus(JobConfiguration jobConfiguration, JobTarget jobTarget, int jobTargetIndex, Stopwatch stopWatch)
{
logger.Info("{0}({0:d}): [{1}/{2}], {3} {4} duration {5:c} ({6} ms)", jobConfiguration.Status, jobTargetIndex, jobConfiguration.Target.Count, jobTarget.Controller, jobTarget.Application, stopWatch.Elapsed, stopWatch.ElapsedMilliseconds);
loggerConsole.Trace("{0}({0:d}): [{1}/{2}], {3} {4} duration {5:c} ({6} ms)", jobConfiguration.Status, jobTargetIndex, jobConfiguration.Target.Count, jobTarget.Controller, jobTarget.Application, stopWatch.Elapsed, stopWatch.ElapsedMilliseconds);
}
public void DisplayJobTargetStartingStatus(JobConfiguration jobConfiguration, JobTarget jobTarget, int jobTargetIndex)
{
logger.Info("{0}({0:d}): [{1}/{2}], {3} {4}({5}) [{6}]", jobConfiguration.Status, jobTargetIndex, jobConfiguration.Target.Count, jobTarget.Controller, jobTarget.Application, jobTarget.ApplicationID, jobTarget.Type);
loggerConsole.Info("{0}({0:d}): [{1}/{2}], {3} {4}({5}) [{6}]", jobConfiguration.Status, jobTargetIndex, jobConfiguration.Target.Count, jobTarget.Controller, jobTarget.Application, jobTarget.ApplicationID, jobTarget.Type);
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
bool reportFolderCleaned = false;
// Process each Controller once
int i = 0;
var controllers = jobConfiguration.Target.GroupBy(t => t.Controller);
foreach (var controllerGroup in controllers)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = controllerGroup.ToList()[0];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 0;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region Controller Settings
loggerConsole.Info("Controller Settings");
List <ControllerSetting> controllerSettingsList = new List <ControllerSetting>();
JArray controllerSettingsArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.ControllerSettingsDataFilePath(jobTarget));
if (controllerSettingsArray != null)
{
foreach (JObject controllerSettingObject in controllerSettingsArray)
{
ControllerSetting controllerSetting = new ControllerSetting();
controllerSetting.Controller = jobTarget.Controller;
controllerSetting.ControllerLink = String.Format(DEEPLINK_CONTROLLER, controllerSetting.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES);
controllerSetting.Name = getStringValueFromJToken(controllerSettingObject, "name");
controllerSetting.Description = getStringValueFromJToken(controllerSettingObject, "description");
controllerSetting.Value = getStringValueFromJToken(controllerSettingObject, "value");
controllerSetting.Updateable = getBoolValueFromJToken(controllerSettingObject, "updateable");
controllerSetting.Scope = getStringValueFromJToken(controllerSettingObject, "scope");
controllerSettingsList.Add(controllerSetting);
}
}
controllerSettingsList = controllerSettingsList.OrderBy(c => c.Name).ToList();
FileIOHelper.WriteListToCSVFile(controllerSettingsList, new ControllerSettingReportMap(), FilePathMap.ControllerSettingsIndexFilePath(jobTarget));
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + controllerSettingsList.Count;
#endregion
#region HTTP Templates
loggerConsole.Info("HTTP Templates");
List <HTTPAlertTemplate> httpTemplatesList = new List <HTTPAlertTemplate>();
JArray httpTemplatesArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.HTTPTemplatesDataFilePath(jobTarget));
JArray httpTemplatesDetailArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.HTTPTemplatesDetailDataFilePath(jobTarget));
if (httpTemplatesArray != null)
{
foreach (JObject httpTemplateObject in httpTemplatesArray)
{
HTTPAlertTemplate httpAlertTemplate = new HTTPAlertTemplate();
httpAlertTemplate.Controller = jobTarget.Controller;
httpAlertTemplate.Name = getStringValueFromJToken(httpTemplateObject, "name");
httpAlertTemplate.Method = getStringValueFromJToken(httpTemplateObject, "method");
httpAlertTemplate.Scheme = getStringValueFromJToken(httpTemplateObject, "scheme");
httpAlertTemplate.Host = getStringValueFromJToken(httpTemplateObject, "host");
httpAlertTemplate.Port = getIntValueFromJToken(httpTemplateObject, "port");
httpAlertTemplate.Path = getStringValueFromJToken(httpTemplateObject, "path");
httpAlertTemplate.Query = getStringValueFromJToken(httpTemplateObject, "query");
httpAlertTemplate.AuthType = getStringValueFromJToken(httpTemplateObject, "authType");
httpAlertTemplate.AuthUsername = getStringValueFromJToken(httpTemplateObject, "authUsername");
httpAlertTemplate.AuthPassword = getStringValueFromJToken(httpTemplateObject, "authPassword");
httpAlertTemplate.Headers = getStringValueOfObjectFromJToken(httpTemplateObject, "headers", true);
if (isTokenPropertyNull(httpTemplateObject, "payloadTemplate") == false)
{
httpAlertTemplate.ContentType = getStringValueFromJToken(httpTemplateObject["payloadTemplate"], "httpRequestActionMediaType");
httpAlertTemplate.FormData = getStringValueOfObjectFromJToken(httpTemplateObject["payloadTemplate"], "formDataPairs", true);
httpAlertTemplate.Payload = getStringValueFromJToken(httpTemplateObject["payloadTemplate"], "payload");
}
httpAlertTemplate.ConnectTimeout = getLongValueFromJToken(httpTemplateObject, "connectTimeoutInMillis");
httpAlertTemplate.SocketTimeout = getLongValueFromJToken(httpTemplateObject, "socketTimeoutInMillis");
httpAlertTemplate.ResponseAny = getStringValueOfObjectFromJToken(httpTemplateObject, "responseMatchCriteriaAnyTemplate");
httpAlertTemplate.ResponseNone = getStringValueOfObjectFromJToken(httpTemplateObject, "responseMatchCriteriaNoneTemplate");
if (httpTemplatesDetailArray != null)
{
JToken httpAlertTemplateToken = httpTemplatesDetailArray.Where(t => t["name"].ToString() == httpAlertTemplate.Name).FirstOrDefault();
if (httpAlertTemplateToken != null)
{
httpAlertTemplate.TemplateID = getLongValueFromJToken(httpAlertTemplateToken, "id");
}
}
httpTemplatesList.Add(httpAlertTemplate);
}
}
httpTemplatesList = httpTemplatesList.OrderBy(c => c.Name).ToList();
FileIOHelper.WriteListToCSVFile(httpTemplatesList, new HTTPAlertTemplateReportMap(), FilePathMap.HTTPTemplatesIndexFilePath(jobTarget));
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + httpTemplatesList.Count;
#endregion
#region Email Templates
loggerConsole.Info("Email Templates");
List <EmailAlertTemplate> emailTemplatesList = new List <EmailAlertTemplate>();
JArray emailTemplatesArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.EmailTemplatesDataFilePath(jobTarget));
JArray emailTemplatesDetailArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.EmailTemplatesDetailDataFilePath(jobTarget));
if (emailTemplatesArray != null)
{
foreach (JObject emailTemplateObject in emailTemplatesArray)
{
EmailAlertTemplate emailAlertTemplate = new EmailAlertTemplate();
emailAlertTemplate.Controller = jobTarget.Controller;
emailAlertTemplate.Name = getStringValueFromJToken(emailTemplateObject, "name");
emailAlertTemplate.OneEmailPerEvent = getBoolValueFromJToken(emailTemplateObject, "oneEmailPerEvent");
emailAlertTemplate.EventLimit = getLongValueFromJToken(emailTemplateObject, "eventClampLimit");
try
{
string[] emails = emailTemplateObject["toRecipients"].Select(s => getStringValueFromJToken(s, "value")).ToArray();
emailAlertTemplate.To = String.Join(";", emails);
}
catch { }
try
{
string[] emails = emailTemplateObject["ccRecipients"].Select(s => getStringValueFromJToken(s, "value")).ToArray();
emailAlertTemplate.CC = String.Join(";", emails);
}
catch { }
try
{
string[] emails = emailTemplateObject["bccRecipients"].Select(s => getStringValueFromJToken(s, "value")).ToArray();
emailAlertTemplate.BCC = String.Join(";", emails);
}
catch { }
try
{
string[] emails = emailTemplateObject["testToRecipients"].Select(s => getStringValueFromJToken(s, "value")).ToArray();
emailAlertTemplate.TestTo = String.Join(";", emails);
}
catch { }
try
{
string[] emails = emailTemplateObject["testCcRecipients"].Select(s => getStringValueFromJToken(s, "value")).ToArray();
emailAlertTemplate.TestCC = String.Join(";", emails);
}
catch { }
try
{
string[] emails = emailTemplateObject["testBccRecipients"].Select(s => getStringValueFromJToken(s, "value")).ToArray();
emailAlertTemplate.TestBCC = String.Join(";", emails);
}
catch { }
emailAlertTemplate.TestLogLevel = getStringValueFromJToken(emailTemplateObject, "testLogLevel");
emailAlertTemplate.Headers = getStringValueOfObjectFromJToken(emailTemplateObject, "headers", true);
emailAlertTemplate.Subject = getStringValueFromJToken(emailTemplateObject, "subject");
emailAlertTemplate.TextBody = getStringValueFromJToken(emailTemplateObject, "textBody");
emailAlertTemplate.HTMLBody = getStringValueFromJToken(emailTemplateObject, "htmlBody");
emailAlertTemplate.IncludeHTMLBody = getBoolValueFromJToken(emailTemplateObject, "includeHtmlBody");
emailAlertTemplate.Properties = getStringValueOfObjectFromJToken(emailTemplateObject, "defaultCustomProperties");
emailAlertTemplate.TestProperties = getStringValueOfObjectFromJToken(emailTemplateObject, "testPropertiesPairs");
emailAlertTemplate.EventTypes = getStringValueOfObjectFromJToken(emailTemplateObject, "eventTypeCountPairs");
if (emailTemplatesDetailArray != null)
{
JToken emailTemplateDetailToken = emailTemplatesDetailArray.Where(t => t["name"].ToString() == emailAlertTemplate.Name).FirstOrDefault();
if (emailTemplateDetailToken != null)
{
emailAlertTemplate.TemplateID = getLongValueFromJToken(emailTemplateDetailToken, "id");
}
}
emailTemplatesList.Add(emailAlertTemplate);
}
}
emailTemplatesList = emailTemplatesList.OrderBy(c => c.Name).ToList();
FileIOHelper.WriteListToCSVFile(emailTemplatesList, new EmailAlertTemplateReportMap(), FilePathMap.EmailTemplatesIndexFilePath(jobTarget));
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + emailTemplatesList.Count;
#endregion
#region Combine All for Report CSV
// If it is the first one, clear out the combined folder
if (reportFolderCleaned == false)
{
FileIOHelper.DeleteFolder(FilePathMap.ControllerSettingsReportFolderPath());
Thread.Sleep(1000);
FileIOHelper.CreateFolder(FilePathMap.ControllerSettingsReportFolderPath());
reportFolderCleaned = true;
}
// Append all the individual report files into one
if (File.Exists(FilePathMap.ControllerSettingsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.ControllerSettingsIndexFilePath(jobTarget)).Length > 0)
{
FileIOHelper.AppendTwoCSVFiles(FilePathMap.ControllerSettingsReportFilePath(), FilePathMap.ControllerSettingsIndexFilePath(jobTarget));
}
if (File.Exists(FilePathMap.HTTPTemplatesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.HTTPTemplatesIndexFilePath(jobTarget)).Length > 0)
{
FileIOHelper.AppendTwoCSVFiles(FilePathMap.HTTPTemplatesReportFilePath(), FilePathMap.HTTPTemplatesIndexFilePath(jobTarget));
}
if (File.Exists(FilePathMap.EmailTemplatesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.EmailTemplatesIndexFilePath(jobTarget)).Length > 0)
{
FileIOHelper.AppendTwoCSVFiles(FilePathMap.EmailTemplatesReportFilePath(), FilePathMap.EmailTemplatesIndexFilePath(jobTarget));
}
#endregion
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
i++;
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
private static MOBILENetworkRequestRule fillNetworkRequestRule(JObject ruleObject, JobTarget jobTarget)
{
MOBILENetworkRequestRule webPageDetectionRule = new MOBILENetworkRequestRule();
webPageDetectionRule.Controller = jobTarget.Controller;
webPageDetectionRule.ControllerLink = String.Format(DEEPLINK_CONTROLLER, jobTarget.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES);
webPageDetectionRule.ApplicationName = jobTarget.Application;
webPageDetectionRule.ApplicationID = jobTarget.ApplicationID;
webPageDetectionRule.ApplicationLink = String.Format(DEEPLINK_MOBILE_APPLICATION, jobTarget.Controller, jobTarget.ParentApplicationID, jobTarget.ApplicationID, DEEPLINK_TIMERANGE_LAST_15_MINUTES);
webPageDetectionRule.RuleName = getStringValueFromJToken(ruleObject, "name");
webPageDetectionRule.IsEnabled = getBoolValueFromJToken(ruleObject, "enabled");
webPageDetectionRule.IsDefault = getBoolValueFromJToken(ruleObject, "isDefault");
if (webPageDetectionRule.IsDefault && webPageDetectionRule.RuleName.Length == 0)
{
webPageDetectionRule.RuleName = "[DEFAULT]";
}
webPageDetectionRule.Priority = getIntValueFromJToken(ruleObject, "priority");
webPageDetectionRule.MatchMobileApp = getMatchRuleDescription((JToken)ruleObject["matchOnMobileApplicationName"]);
webPageDetectionRule.MatchURL = getMatchRuleDescription((JToken)ruleObject["matchOnURL"]);
webPageDetectionRule.MatchIPAddress = getMatchRuleDescription((JToken)ruleObject["matchOnIpAddressMasks"]);
webPageDetectionRule.MatchUserAgent = getMatchRuleDescription((JToken)ruleObject["matchOnUserAgent"]);
webPageDetectionRule.MatchUserAgentType = getMatchRuleDescription((JToken)ruleObject["matchOnUserAgentType"]);
if (isTokenPropertyNull(ruleObject, "pageNamingConfig") == false)
{
JObject pageNamingConfigObject = (JObject)ruleObject["pageNamingConfig"];
webPageDetectionRule.UseProtocol = getBoolValueFromJToken(pageNamingConfigObject, "useProtocol");
webPageDetectionRule.UseDomain = getBoolValueFromJToken(pageNamingConfigObject, "useDomainName");
webPageDetectionRule.UseURL = getBoolValueFromJToken(pageNamingConfigObject, "useURL");
webPageDetectionRule.UseRegex = getBoolValueFromJToken(pageNamingConfigObject, "useRegex");
webPageDetectionRule.UseHTTP = getBoolValueFromJToken(pageNamingConfigObject, "useHttpMethod");
webPageDetectionRule.NamingType = getStringValueFromJToken(pageNamingConfigObject, "type");
webPageDetectionRule.AnchorType = getStringValueFromJToken(pageNamingConfigObject, "anchorType");
webPageDetectionRule.UrlSegments = getStringValueOfObjectFromJToken(pageNamingConfigObject, "urlMatchSegments", true);
webPageDetectionRule.AnchorSegments = getStringValueOfObjectFromJToken(pageNamingConfigObject, "anchorMatchSegments", true);
webPageDetectionRule.RegexGroups = getStringValueOfObjectFromJToken(pageNamingConfigObject, "regexGroupConfig", true);
webPageDetectionRule.QueryStrings = getStringValueOfObjectFromJToken(pageNamingConfigObject, "queryStringMatch", true).Replace("[]", "");
webPageDetectionRule.DomainNameType = getStringValueFromJToken(pageNamingConfigObject, "domainNameType");
}
return(webPageDetectionRule);
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(programOptions, jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_MOBILE) == 0)
{
logger.Warn("No {0} targets to process", APPLICATION_TYPE_MOBILE);
loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_MOBILE);
return(true);
}
#region Template comparisons
if (jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE.Controller == BLANK_APPLICATION_CONTROLLER &&
jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE.Application == BLANK_APPLICATION_MOBILE)
{
jobConfiguration.Target.Add(jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE);
}
else
{
// Check if there is a valid reference application
JobTarget jobTargetReferenceApp = jobConfiguration.Target.Where(t =>
t.Type == APPLICATION_TYPE_MOBILE &&
String.Compare(t.Controller, jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE.Controller, StringComparison.InvariantCultureIgnoreCase) == 0 &&
String.Compare(t.Application, jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE.Application, StringComparison.InvariantCultureIgnoreCase) == 0).FirstOrDefault();
if (jobTargetReferenceApp == null)
{
// No valid reference, fall back to comparing against template
logger.Warn("Unable to find reference target {0}, will index default template", jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE);
loggerConsole.Warn("Unable to find reference target {0}, will index default template", jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE);
jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE.Controller = BLANK_APPLICATION_CONTROLLER;
jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE.Application = BLANK_APPLICATION_MOBILE;
jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE.Type = APPLICATION_TYPE_MOBILE;
jobConfiguration.Target.Add(jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE);
}
}
#endregion
bool reportFolderCleaned = false;
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_MOBILE)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region Preload list of detected entities
// For later cross-reference
List <ControllerApplication> controllerApplicationList = FileIOHelper.ReadListFromCSVFile <ControllerApplication>(FilePathMap.ControllerApplicationsIndexFilePath(jobTarget), new ControllerApplicationReportMap());
#endregion
#region Application Summary
MOBILEApplicationConfiguration applicationConfiguration = new MOBILEApplicationConfiguration();
loggerConsole.Info("Application Summary");
applicationConfiguration.Controller = jobTarget.Controller;
applicationConfiguration.ControllerLink = String.Format(DEEPLINK_CONTROLLER, applicationConfiguration.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES);
applicationConfiguration.ApplicationName = jobTarget.Application;
applicationConfiguration.ApplicationID = jobTarget.ApplicationID;
applicationConfiguration.ApplicationLink = String.Format(DEEPLINK_APM_APPLICATION, applicationConfiguration.Controller, applicationConfiguration.ApplicationID, DEEPLINK_TIMERANGE_LAST_15_MINUTES);
if (controllerApplicationList != null)
{
ControllerApplication controllerApplication = controllerApplicationList.Where(a => a.Type == APPLICATION_TYPE_MOBILE && a.ApplicationID == jobTarget.ApplicationID).FirstOrDefault();
if (controllerApplication != null)
{
applicationConfiguration.ApplicationDescription = controllerApplication.Description;
}
}
// Application Key
JObject appKeyObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.MOBILEApplicationKeyDataFilePath(jobTarget));
if (appKeyObject != null)
{
applicationConfiguration.ApplicationKey = getStringValueFromJToken(appKeyObject, "appKey");
}
// Monitoring State
string monitoringState = FileIOHelper.ReadFileFromPath(FilePathMap.MOBILEApplicationMonitoringStateDataFilePath(jobTarget));
if (monitoringState != String.Empty)
{
bool parsedBool = false;
Boolean.TryParse(monitoringState, out parsedBool);
applicationConfiguration.IsEnabled = parsedBool;
}
// Configuration Settings
JObject configurationSettingsObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.MOBILEAgentPageSettingsRulesDataFilePath(jobTarget));
if (configurationSettingsObject != null)
{
if (isTokenPropertyNull(configurationSettingsObject, "thresholds") == false)
{
applicationConfiguration.SlowThresholdType = getStringValueFromJToken(configurationSettingsObject["thresholds"]["slowThreshold"], "type");
applicationConfiguration.SlowThreshold = getIntValueFromJToken(configurationSettingsObject["thresholds"]["slowThreshold"], "value");
applicationConfiguration.VerySlowThresholdType = getStringValueFromJToken(configurationSettingsObject["thresholds"]["verySlowThreshold"], "type");
applicationConfiguration.VerySlowThreshold = getIntValueFromJToken(configurationSettingsObject["thresholds"]["verySlowThreshold"], "value");
applicationConfiguration.StallThresholdType = getStringValueFromJToken(configurationSettingsObject["thresholds"]["stallThreshold"], "type");
applicationConfiguration.StallThreshold = getIntValueFromJToken(configurationSettingsObject["thresholds"]["stallThreshold"], "value");
}
applicationConfiguration.Percentiles = getStringValueOfObjectFromJToken(configurationSettingsObject, "percentileMetrics", true);
applicationConfiguration.SessionTimeout = getIntValueFromJToken(configurationSettingsObject["sessionsMonitor"], "sessionTimeoutMins");
applicationConfiguration.CrashThreshold = getIntValueFromJToken(configurationSettingsObject["crashAlerts"], "threshold");
applicationConfiguration.IsIPDisplayed = getBoolValueFromJToken(configurationSettingsObject, "ipAddressDisplayed");
applicationConfiguration.EnableScreenshot = getBoolValueFromJToken(configurationSettingsObject["agentConfigData"], "enableScreenshot");
applicationConfiguration.AutoScreenshot = getBoolValueFromJToken(configurationSettingsObject["agentConfigData"], "autoScreenshot");
applicationConfiguration.UseCellular = getBoolValueFromJToken(configurationSettingsObject["agentConfigData"], "screenshotUseCellular");
}
#endregion
#region Network Requests
List <MOBILENetworkRequestRule> networkRequestRulesList = new List <MOBILENetworkRequestRule>(128);
JObject networkRequestRulesObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.MOBILEAgentNetworkRequestsRulesDataFilePath(jobTarget));
if (networkRequestRulesObject != null)
{
if (isTokenPropertyNull(networkRequestRulesObject, "customNamingIncludeRules") == false)
{
JArray includeRulesArray = (JArray)networkRequestRulesObject["customNamingIncludeRules"];
foreach (JObject includeRuleObject in includeRulesArray)
{
MOBILENetworkRequestRule networkRequestRule = fillNetworkRequestRule(includeRuleObject, jobTarget);
if (networkRequestRule != null)
{
networkRequestRule.DetectionType = "INCLUDE";
networkRequestRulesList.Add(networkRequestRule);
}
}
}
if (isTokenPropertyNull(networkRequestRulesObject, "customNamingExcludeRules") == false)
{
JArray excludeRulesArray = (JArray)networkRequestRulesObject["customNamingExcludeRules"];
foreach (JObject excludeRuleObject in excludeRulesArray)
{
MOBILENetworkRequestRule networkRequestRule = fillNetworkRequestRule(excludeRuleObject, jobTarget);
if (networkRequestRule != null)
{
networkRequestRule.DetectionType = "EXCLUDE";
networkRequestRulesList.Add(networkRequestRule);
}
}
}
}
// Sort them
networkRequestRulesList = networkRequestRulesList.OrderBy(o => o.DetectionType).ThenBy(o => o.Priority).ToList();
FileIOHelper.WriteListToCSVFile(networkRequestRulesList, new MOBILENetworkRequestRuleReportMap(), FilePathMap.MOBILENetworkRequestRulesIndexFilePath(jobTarget));
loggerConsole.Info("Completed {0} Rules", networkRequestRulesList.Count);
#endregion
#region Application Settings
if (networkRequestRulesList != null)
{
applicationConfiguration.NumNetworkRulesInclude = networkRequestRulesList.Count(r => r.DetectionType == "INCLUDE");
applicationConfiguration.NumNetworkRulesExclude = networkRequestRulesList.Count(r => r.DetectionType == "EXCLUDE");
}
List <MOBILEApplicationConfiguration> applicationConfigurationsList = new List <MOBILEApplicationConfiguration>(1);
applicationConfigurationsList.Add(applicationConfiguration);
FileIOHelper.WriteListToCSVFile(applicationConfigurationsList, new MOBILEApplicationConfigurationReportMap(), FilePathMap.MOBILEApplicationConfigurationIndexFilePath(jobTarget));
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + applicationConfigurationsList.Count;
#endregion
#region Combine All for Report CSV
// If it is the first one, clear out the combined folder
if (reportFolderCleaned == false)
{
FileIOHelper.DeleteFolder(FilePathMap.MOBILEConfigurationReportFolderPath());
Thread.Sleep(1000);
FileIOHelper.CreateFolder(FilePathMap.WEBConfigurationReportFolderPath());
reportFolderCleaned = true;
}
// Append all the individual report files into one
if (File.Exists(FilePathMap.MOBILEApplicationConfigurationIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.MOBILEApplicationConfigurationIndexFilePath(jobTarget)).Length > 0)
{
FileIOHelper.AppendTwoCSVFiles(FilePathMap.MOBILEApplicationConfigurationReportFilePath(), FilePathMap.MOBILEApplicationConfigurationIndexFilePath(jobTarget));
}
if (File.Exists(FilePathMap.MOBILENetworkRequestRulesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.MOBILENetworkRequestRulesIndexFilePath(jobTarget)).Length > 0)
{
FileIOHelper.AppendTwoCSVFiles(FilePathMap.MOBILENetworkRequestRulesReportFilePath(), FilePathMap.MOBILENetworkRequestRulesIndexFilePath(jobTarget));
}
#endregion
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
// Remove all templates from the list
jobConfiguration.Target.RemoveAll(t => t.Controller == BLANK_APPLICATION_CONTROLLER);
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
private int extractSnapshots(
JobConfiguration jobConfiguration,
JobTarget jobTarget,
ControllerApi controllerApi,
List <JToken> entityList,
List <AppDRESTTier> tiersNodeJSList,
bool progressToConsole)
{
int j = 0;
foreach (JToken snapshot in entityList)
{
// Only do first in chain
if ((bool)snapshot["firstInChain"] == true)
{
logger.Info("Retrieving snapshot for Application {0}, Tier {1}, Business Transaction {2}, RequestGUID {3}", jobTarget.Application, snapshot["applicationComponentName"], snapshot["businessTransactionName"], snapshot["requestGUID"]);
#region Target step variables
DateTime snapshotTime = UnixTimeHelper.ConvertFromUnixTimestamp((long)snapshot["serverStartTime"]);
string snapshotFolderPath = FilePathMap.SnapshotDataFolderPath(
jobTarget,
snapshot["applicationComponentName"].ToString(), (long)snapshot["applicationComponentId"],
snapshot["businessTransactionName"].ToString(), (long)snapshot["businessTransactionId"],
snapshotTime,
snapshot["userExperience"].ToString(),
snapshot["requestGUID"].ToString());
// Must strip out the milliseconds, because the segment list retrieval doesn't seem to like them in the datetimes
DateTime snapshotTimeFrom = snapshotTime.AddMinutes(-30).AddMilliseconds(snapshotTime.Millisecond * -1);
DateTime snapshotTimeTo = snapshotTime.AddMinutes(30).AddMilliseconds(snapshotTime.Millisecond * -1);
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(snapshotTimeFrom);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(snapshotTimeTo);
int differenceInMinutes = (int)(snapshotTimeTo - snapshotTimeFrom).TotalMinutes;
#endregion
#region Get Snapshot Flowmap
// Get snapshot flow map
// Commenting this out until the time I decide to build visual representation of it, until then it is not needed
//string snapshotFlowmapDataFilePath = Path.Combine(snapshotFolderPath, EXTRACT_SNAPSHOT_FLOWMAP_FILE_NAME);
//if (File.Exists(snapshotFlowmapDataFilePath) == false)
//{
// string snapshotFlowmapJson = controllerApi.GetFlowmapSnapshot(jobTarget.ApplicationID, (int)snapshot["businessTransactionId"], snapshot["requestGUID"].ToString(), fromTimeUnix, toTimeUnix, differenceInMinutes);
// if (snapshotFlowmapJson != String.Empty) FileIOHelper.SaveFileToPath(snapshotFlowmapJson, snapshotFlowmapDataFilePath);
//}
#endregion
#region Get List of Segments
// Get list of segments
string snapshotSegmentsDataFilePath = FilePathMap.SnapshotSegmentsDataFilePath(snapshotFolderPath);
if (File.Exists(snapshotSegmentsDataFilePath) == false)
{
string snapshotSegmentsJson = controllerApi.GetSnapshotSegments(snapshot["requestGUID"].ToString(), snapshotTimeFrom, snapshotTimeTo, differenceInMinutes);
if (snapshotSegmentsJson != String.Empty)
{
FileIOHelper.SaveFileToPath(snapshotSegmentsJson, snapshotSegmentsDataFilePath);
}
}
#endregion
#region Get Details for Each Segment
JArray snapshotSegmentsList = FileIOHelper.LoadJArrayFromFile(snapshotSegmentsDataFilePath);
if (snapshotSegmentsList != null)
{
// Get details for segment
foreach (JToken snapshotSegment in snapshotSegmentsList)
{
string snapshotSegmentDataFilePath = FilePathMap.SnapshotSegmentDataFilePath(snapshotFolderPath, snapshotSegment["id"].ToString());
if (File.Exists(snapshotSegmentDataFilePath) == false)
{
string snapshotSegmentJson = controllerApi.GetSnapshotSegmentDetails((long)snapshotSegment["id"], fromTimeUnix, toTimeUnix, differenceInMinutes);
if (snapshotSegmentJson != String.Empty)
{
FileIOHelper.SaveFileToPath(snapshotSegmentJson, snapshotSegmentDataFilePath);
}
}
}
// Get errors for segment
foreach (JToken snapshotSegment in snapshotSegmentsList)
{
string snapshotSegmentErrorFilePath = FilePathMap.SnapshotSegmentErrorDataFilePath(snapshotFolderPath, snapshotSegment["id"].ToString());
if (File.Exists(snapshotSegmentErrorFilePath) == false && (bool)snapshotSegment["errorOccurred"] == true)
{
string snapshotSegmentJson = controllerApi.GetSnapshotSegmentErrors((long)snapshotSegment["id"], fromTimeUnix, toTimeUnix, differenceInMinutes);
if (snapshotSegmentJson != String.Empty)
{
// "[ ]" == empty data. Don't create the file
if (snapshotSegmentJson.Length > 3)
{
FileIOHelper.SaveFileToPath(snapshotSegmentJson, snapshotSegmentErrorFilePath);
}
}
}
}
// Get call graphs for segment
foreach (JToken snapshotSegment in snapshotSegmentsList)
{
string snapshotSegmentCallGraphFilePath = FilePathMap.SnapshotSegmentCallGraphDataFilePath(snapshotFolderPath, snapshotSegment["id"].ToString());
if (File.Exists(snapshotSegmentCallGraphFilePath) == false && ((bool)snapshotSegment["fullCallgraph"] == true || (bool)snapshotSegment["delayedCallGraph"] == true))
{
// If the tier is Node.JS, the call graphs come from Process Snapshot
bool getProcessCallGraph = false;
string processRequestGUID = String.Empty;
if (tiersNodeJSList != null && tiersNodeJSList.Count > 0)
{
// Is this a Node.JS tier?
if (tiersNodeJSList.Count(t => t.id == (long)snapshotSegment["applicationComponentId"]) > 0)
{
// Yes, it is
// Is there a process snapshot? Check Transaction Properties for its value
string snapshotSegmentDataFilePath = FilePathMap.SnapshotSegmentDataFilePath(snapshotFolderPath, snapshotSegment["id"].ToString());
JObject snapshotSegmentDetail = FileIOHelper.LoadJObjectFromFile(snapshotSegmentDataFilePath);
if (snapshotSegmentDetail != null)
{
if (snapshotSegmentDetail["transactionProperties"].HasValues == true)
{
foreach (JToken transactionPropertyToken in snapshotSegmentDetail["transactionProperties"])
{
if (transactionPropertyToken["name"].ToString() == "Process Snapshot GUIDs")
{
getProcessCallGraph = true;
processRequestGUID = transactionPropertyToken["value"].ToString();
break;
}
}
}
}
}
}
// Ok, now either get call graph the usual way or process snapshot call graph
if (getProcessCallGraph == true && processRequestGUID.Length > 0)
{
string snapshotSegmentJson = controllerApi.GetProcessSnapshotCallGraph(processRequestGUID, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (snapshotSegmentJson != String.Empty)
{
FileIOHelper.SaveFileToPath(snapshotSegmentJson, snapshotSegmentCallGraphFilePath);
}
}
else
{
string snapshotSegmentJson = controllerApi.GetSnapshotSegmentCallGraph((long)snapshotSegment["id"], fromTimeUnix, toTimeUnix, differenceInMinutes);
if (snapshotSegmentJson != String.Empty)
{
FileIOHelper.SaveFileToPath(snapshotSegmentJson, snapshotSegmentCallGraphFilePath);
}
}
}
}
}
#endregion
}
if (progressToConsole == true)
{
j++;
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
}
return(entityList.Count);
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region Target state check
if (jobTarget.Status != JobTargetStatus.ConfigurationValid)
{
loggerConsole.Trace("Target in invalid state {0}, skipping", jobTarget.Status);
continue;
}
#endregion
#region Target step variables
// Set up controller access
ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
// Login into private API
controllerApi.PrivateApiLogin();
#endregion
#region Get list of Snapshots in time ranges
loggerConsole.Info("Extract List of Snapshots ({0} time ranges)", jobConfiguration.Input.HourlyTimeRanges.Count);
// Get list of snapshots in each time range
int totalSnapshotsFound = 0;
foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges)
{
logger.Info("Extract List of Snapshots from {0:o} to {1:o}", jobTimeRange.From, jobTimeRange.To);
loggerConsole.Info("Extract List of Snapshots from {0:G} to {1:G}", jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime());
string snapshotsDataFilePath = FilePathMap.SnapshotsDataFilePath(jobTarget, jobTimeRange);
int differenceInMinutes = (int)(jobTimeRange.To - jobTimeRange.From).TotalMinutes;
if (File.Exists(snapshotsDataFilePath) == false)
{
JArray listOfSnapshots = new JArray();
// Extract snapshot list
long serverCursorId = 0;
string serverCursorIdType = String.Empty;
do
{
string snapshotsJSON = String.Empty;
if (serverCursorId == 0)
{
// Extract first page of snapshots
snapshotsJSON = controllerApi.GetListOfSnapshotsFirstPage(jobTarget.ApplicationID, jobTimeRange.From, jobTimeRange.To, differenceInMinutes, SNAPSHOTS_QUERY_PAGE_SIZE);
}
else
{
// If there are more snapshots on the server, the server cursor would be non-0
switch (serverCursorIdType)
{
case "scrollId":
// Sometimes - >4.3.3? the value of scroll is in scrollId, not rsdScrollId
// "serverCursor" : {
// "scrollId" : 1509543646696
// }
snapshotsJSON = controllerApi.GetListOfSnapshotsNextPage_Type_scrollId(jobTarget.ApplicationID, jobTimeRange.From, jobTimeRange.To, differenceInMinutes, SNAPSHOTS_QUERY_PAGE_SIZE, serverCursorId);
break;
case "rsdScrollId":
// "serverCursor" : {
// "rsdScrollId" : 1509543646696
// }
snapshotsJSON = controllerApi.GetListOfSnapshotsNextPage_Type_rsdScrollId(jobTarget.ApplicationID, jobTimeRange.From, jobTimeRange.To, differenceInMinutes, SNAPSHOTS_QUERY_PAGE_SIZE, serverCursorId);
break;
case "fetchMoreDataHandle":
// Seen this on 4.2.3.0 Controller. Maybe that's how it used to be?
// "fetchMoreDataHandle":1509626881987
// Can't seem to make it load more than 600 items
snapshotsJSON = controllerApi.GetListOfSnapshotsNextPage_Type_handle(jobTarget.ApplicationID, jobTimeRange.From, jobTimeRange.To, differenceInMinutes, SNAPSHOTS_QUERY_PAGE_SIZE, serverCursorId);
break;
default:
logger.Warn("Unknown type of serverCursorIdType={0}, not going to retrieve any snapshots", serverCursorIdType);
break;
}
}
// Assume we have no more pages
serverCursorId = 0;
// Process retrieved snapshots and check if we actually have more pages
if (snapshotsJSON != String.Empty)
{
Console.Write(".");
// Load snapshots into array
JObject snapshotsParsed = JObject.Parse(snapshotsJSON);
JArray snapshots = (JArray)snapshotsParsed["requestSegmentDataListItems"];
foreach (JObject snapshot in snapshots)
{
listOfSnapshots.Add(snapshot);
}
// Check whether we have more snapshots and if yes, get continuation type and cursor ID
JToken fetchMoreDataHandleObj = snapshotsParsed["fetchMoreDataHandle"];
JToken serverCursorObj = snapshotsParsed["serverCursor"];
if (serverCursorObj != null)
{
JToken scrollIdObj = serverCursorObj["scrollId"];
JToken rsdScrollIdObj = serverCursorObj["rsdScrollId"];
if (scrollIdObj != null)
{
serverCursorIdType = "scrollId";
// Parse the cursor ID
if (Int64.TryParse(scrollIdObj.ToString(), out serverCursorId) == false)
{
// Nope, not going to go forward
serverCursorId = 0;
}
}
else if (rsdScrollIdObj != null)
{
serverCursorIdType = "rsdScrollId";
// Parse the cursor ID
if (Int64.TryParse(rsdScrollIdObj.ToString(), out serverCursorId) == false)
{
// Nope, not going to go forward
serverCursorId = 0;
}
}
}
else if (fetchMoreDataHandleObj != null)
{
serverCursorIdType = "fetchMoreDataHandle";
// Parse the cursor ID
if (Int64.TryParse(fetchMoreDataHandleObj.ToString(), out serverCursorId) == false)
{
// Nope, not going to go forward
serverCursorId = 0;
}
}
else
{
logger.Warn("Snapshot list retrival call unexpectedly did not have any evidence of continuation CursorId");
}
logger.Info("Retrieved snapshots from Controller {0}, Application {1}, From {2:o}, To {3:o}', number of snapshots {4}, continuation type {5}, continuation CursorId {6}", jobTarget.Controller, jobTarget.Application, jobTimeRange.From, jobTimeRange.To, snapshots.Count, serverCursorIdType, serverCursorId);
// Move to next loop
Console.Write("+{0}", listOfSnapshots.Count);
}
}while (serverCursorId > 0);
Console.WriteLine();
FileIOHelper.WriteJArrayToFile(listOfSnapshots, snapshotsDataFilePath);
totalSnapshotsFound = totalSnapshotsFound + listOfSnapshots.Count;
logger.Info("{0} snapshots from {1:o} to {2:o}", listOfSnapshots.Count, jobTimeRange.From, jobTimeRange.To);
loggerConsole.Info("{0} snapshots from {1:G} to {2:G}", listOfSnapshots.Count, jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime());
}
}
logger.Info("{0} snapshots in all time ranges", totalSnapshotsFound);
loggerConsole.Info("{0} snapshots in all time ranges", totalSnapshotsFound);
#endregion
#region Get individual Snapshots
// Extract individual snapshots
loggerConsole.Info("Extract Individual Snapshots");
List <AppDRESTTier> tiersList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTTier>(FilePathMap.TiersDataFilePath(jobTarget));
List <AppDRESTBusinessTransaction> businessTransactionsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBusinessTransaction>(FilePathMap.BusinessTransactionsDataFilePath(jobTarget));
// Identify Node.JS tiers that will extact call graph using a different call
List <AppDRESTTier> tiersNodeJSList = null;
if (tiersList != null)
{
tiersNodeJSList = tiersList.Where(t => t.agentType == "NODEJS_APP_AGENT").ToList();
}
// Process each hour at a time
foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges)
{
string snapshotsDataFilePath = FilePathMap.SnapshotsDataFilePath(jobTarget, jobTimeRange);
JArray listOfSnapshotsInHour = FileIOHelper.LoadJArrayFromFile(snapshotsDataFilePath);
if (listOfSnapshotsInHour != null && listOfSnapshotsInHour.Count > 0)
{
logger.Info("Filter Snapshots {0:o} to {1:o} ({2} snapshots)", jobTimeRange.From, jobTimeRange.To, listOfSnapshotsInHour.Count);
loggerConsole.Info("Filter Snapshots {0:G} to {1:G} ({2} snapshots)", jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime(), listOfSnapshotsInHour.Count);
// Filter the list of snapshots based on SnapshotSelectionCriteria
List <JToken> listOfSnapshotsInHourFiltered = new List <JToken>(listOfSnapshotsInHour.Count);
foreach (JToken snapshotToken in listOfSnapshotsInHour)
{
logger.Trace("Considering filtering snapshot requestGUID={0}, firstInChain={1}, userExperience={2}, fullCallgraph={3}, delayedCallGraph={4}, applicationComponentName={5}, businessTransactionName={6}",
snapshotToken["requestGUID"],
snapshotToken["firstInChain"],
snapshotToken["userExperience"],
snapshotToken["fullCallgraph"],
snapshotToken["delayedCallGraph"],
snapshotToken["applicationComponentName"],
snapshotToken["businessTransactionName"]);
// Only grab first in chain snapshots
if ((bool)snapshotToken["firstInChain"] == false)
{
continue;
}
// Filter user experience
switch (snapshotToken["userExperience"].ToString())
{
case "NORMAL":
if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Normal != true)
{
continue;
}
break;
case "SLOW":
if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Slow != true)
{
continue;
}
break;
case "VERY_SLOW":
if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.VerySlow != true)
{
continue;
}
break;
case "STALL":
if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Stall != true)
{
continue;
}
break;
case "ERROR":
if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Error != true)
{
continue;
}
break;
default:
// Not sure what kind of beast it is
continue;
}
// Filter call graph
if ((bool)snapshotToken["fullCallgraph"] == true)
{
if (jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType.Full != true)
{
continue;
}
}
else if ((bool)snapshotToken["delayedCallGraph"] == true)
{
if (jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType.Partial != true)
{
continue;
}
}
else
{
if (jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType.None != true)
{
continue;
}
}
// Filter Tier type
if (jobConfiguration.Input.SnapshotSelectionCriteria.TierType.All != true)
{
if (tiersList != null)
{
AppDRESTTier tier = tiersList.Where(t => t.id == (long)snapshotToken["applicationComponentId"]).FirstOrDefault();
if (tier != null)
{
PropertyInfo pi = jobConfiguration.Input.SnapshotSelectionCriteria.TierType.GetType().GetProperty(tier.agentType);
if (pi != null)
{
if ((bool)pi.GetValue(jobConfiguration.Input.SnapshotSelectionCriteria.TierType) == false)
{
continue;
}
}
}
}
}
// Filter BT type
if (jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactionType.All != true)
{
if (businessTransactionsList != null)
{
AppDRESTBusinessTransaction businessTransaction = businessTransactionsList.Where(b => b.id == (long)snapshotToken["businessTransactionId"] && b.tierId == (long)snapshotToken["applicationComponentId"]).FirstOrDefault();
if (businessTransaction != null)
{
PropertyInfo pi = jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactionType.GetType().GetProperty(businessTransaction.entryPointType);
if (pi != null)
{
if ((bool)pi.GetValue(jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactionType) == false)
{
continue;
}
}
}
}
}
// Filter Tier name
bool tierNameMatch = false;
if (jobConfiguration.Input.SnapshotSelectionCriteria.Tiers.Length == 0)
{
tierNameMatch = true;
}
foreach (string matchCriteria in jobConfiguration.Input.SnapshotSelectionCriteria.Tiers)
{
if (matchCriteria.Length > 0)
{
// Try straight up string compare first
if (String.Compare(snapshotToken["applicationComponentName"].ToString(), matchCriteria, true) == 0)
{
tierNameMatch = true;
break;
}
// Try regex compare second
Regex regexQuery = new Regex(matchCriteria, RegexOptions.IgnoreCase);
Match regexMatch = regexQuery.Match(snapshotToken["applicationComponentName"].ToString());
if (regexMatch.Success == true && regexMatch.Index == 0)
{
tierNameMatch = true;
break;
}
}
}
if (tierNameMatch == false)
{
continue;
}
// Filter BT name
bool businessTransactionNameMatch = false;
if (jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactions.Length == 0)
{
businessTransactionNameMatch = true;
}
foreach (string matchCriteria in jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactions)
{
if (matchCriteria.Length > 0)
{
// Try straight up string compare first
if (String.Compare(snapshotToken["businessTransactionName"].ToString(), matchCriteria, true) == 0)
{
businessTransactionNameMatch = true;
break;
}
// Try regex compare second
Regex regexQuery = new Regex(matchCriteria, RegexOptions.IgnoreCase);
Match regexMatch = regexQuery.Match(snapshotToken["businessTransactionName"].ToString());
if (regexMatch.Success == true && regexMatch.Index == 0)
{
businessTransactionNameMatch = true;
break;
}
}
}
if (businessTransactionNameMatch == false)
{
continue;
}
// If we got here, then the snapshot passed the filter
logger.Trace("Keeping snapshot requestGUID={0}, firstInChain={1}, userExperience={2}, fullCallgraph={3}, delayedCallGraph={4}, applicationComponentName={5}, businessTransactionName={6}",
snapshotToken["requestGUID"],
snapshotToken["firstInChain"],
snapshotToken["userExperience"],
snapshotToken["fullCallgraph"],
snapshotToken["delayedCallGraph"],
snapshotToken["applicationComponentName"],
snapshotToken["businessTransactionName"]);
listOfSnapshotsInHourFiltered.Add(snapshotToken);
}
logger.Info("Total Snapshots {0:o} to {1:o} is {2}, after filtered {3}", jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime(), listOfSnapshotsInHour.Count, listOfSnapshotsInHourFiltered.Count);
// Now extract things
logger.Info("Extract Snapshots {0:o} to {1:o} ({2} snapshots)", jobTimeRange.From, jobTimeRange.To, listOfSnapshotsInHourFiltered.Count);
loggerConsole.Info("Extract Snapshots {0:G} to {1:G} ({2} snapshots)", jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime(), listOfSnapshotsInHourFiltered.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + listOfSnapshotsInHourFiltered.Count;
int numSnapshots = 0;
if (programOptions.ProcessSequentially == false)
{
var listOfSnapshotsInHourChunks = listOfSnapshotsInHourFiltered.BreakListIntoChunks(SNAPSHOTS_EXTRACT_NUMBER_OF_ENTITIES_TO_PROCESS_PER_THREAD);
Parallel.ForEach <List <JToken>, int>(
listOfSnapshotsInHourChunks,
new ParallelOptions {
MaxDegreeOfParallelism = SNAPSHOTS_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(listOfSnapshotsInHourChunk, loop, subtotal) =>
{
// Set up controller access
ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
// Login into private API
controllerApiParallel.PrivateApiLogin();
subtotal += extractSnapshots(jobConfiguration, jobTarget, controllerApiParallel, listOfSnapshotsInHourChunk, tiersNodeJSList, false);
return(subtotal);
},
(finalResult) =>
{
Interlocked.Add(ref numSnapshots, finalResult);
Console.Write("[{0}].", numSnapshots);
}
);
}
else
{
numSnapshots = extractSnapshots(jobConfiguration, jobTarget, controllerApi, listOfSnapshotsInHourFiltered, tiersNodeJSList, true);
}
loggerConsole.Info("{0} snapshots", numSnapshots);
}
}
#endregion
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
// Process each Controller once
int i = 0;
var controllers = jobConfiguration.Target.GroupBy(t => t.Controller);
foreach (var controllerGroup in controllers)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = controllerGroup.ToList()[0];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
#region Controller
loggerConsole.Info("Controller Version");
string controllerVersionXML = controllerApi.GetControllerVersion();
if (controllerVersionXML != String.Empty)
{
FileIOHelper.SaveFileToPath(controllerVersionXML, FilePathMap.ControllerVersionDataFilePath(jobTarget));
}
#endregion
#region APM Applications
loggerConsole.Info("APM Applications");
string applicationsJSON = controllerApi.GetAPMApplications();
if (applicationsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationsJSON, FilePathMap.APMApplicationsDataFilePath(jobTarget));
}
#endregion
#region Applications
loggerConsole.Info("All Applications");
controllerApi.PrivateApiLogin();
string applicationsAllJSON = controllerApi.GetAllApplicationsAllTypes();
if (applicationsAllJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationsAllJSON, FilePathMap.AllApplicationsDataFilePath(jobTarget));
}
#endregion
#region Mobile Applications
loggerConsole.Info("Mobile Applications");
string applicationsMobileJSON = controllerApi.GetMOBILEApplications();
if (applicationsMobileJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationsMobileJSON, FilePathMap.MOBILEApplicationsDataFilePath(jobTarget));
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
i++;
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(programOptions, jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_APM) == 0)
{
logger.Warn("No {0} targets to process", APPLICATION_TYPE_APM);
loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_APM);
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_APM)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region Target step variables
int numEntitiesTotal = 0;
#endregion
#region Prepare time range
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To);
long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000);
#endregion
ParallelOptions parallelOptions = new ParallelOptions();
if (programOptions.ProcessSequentially == true)
{
parallelOptions.MaxDegreeOfParallelism = 1;
}
Parallel.Invoke(parallelOptions,
() =>
{
#region Application
loggerConsole.Info("Extract Flowmap for Application");
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApi.PrivateApiLogin();
if (File.Exists(FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange)) == false)
{
string flowmapJson = controllerApi.GetFlowmapApplication(jobTarget.ApplicationID, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
loggerConsole.Info("Completed Application");
}
// Removing as this doesn't seem to be a helpful report
//JobTimeRange jobTimeRangeLast = jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1];
//int differenceInMinutesForLastTimeRange = (int)((jobTimeRangeLast.To - jobTimeRangeLast.From).TotalMinutes);
//loggerConsole.Info("Extract Flowmap for Application in each minute in in last timerange ({0} minutes)", differenceInMinutesForLastTimeRange);
//int j = 0;
//Parallel.For(0,
// differenceInMinutesForLastTimeRange,
// parallelOptions,
// () => 0,
// (minute, loop, subtotal) =>
// {
// using (ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
// {
// controllerApiLocal.PrivateApiLogin();
// JobTimeRange thisMinuteJobTimeRange = new JobTimeRange();
// thisMinuteJobTimeRange.From = jobTimeRangeLast.From.AddMinutes(minute);
// thisMinuteJobTimeRange.To = jobTimeRangeLast.From.AddMinutes(minute + 1);
// long fromTimeUnixLocal = UnixTimeHelper.ConvertToUnixTimestamp(thisMinuteJobTimeRange.From);
// long toTimeUnixLocal = UnixTimeHelper.ConvertToUnixTimestamp(thisMinuteJobTimeRange.To);
// long differenceInMinutesLocal = 1;
// if (File.Exists(FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, thisMinuteJobTimeRange)) == false)
// {
// string flowmapJson = controllerApiLocal.GetFlowmapApplication(jobTarget.ApplicationID, fromTimeUnixLocal, toTimeUnixLocal, differenceInMinutesLocal);
// if (flowmapJson != String.Empty) FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, thisMinuteJobTimeRange));
// }
// return 1;
// }
// },
// (finalResult) =>
// {
// Interlocked.Add(ref j, finalResult);
// if (j % 10 == 0)
// {
// Console.Write("[{0}].", j);
// }
// }
//);
//loggerConsole.Info("Completed Application in each minute {0} minutes", differenceInMinutesForLastTimeRange);
Interlocked.Add(ref numEntitiesTotal, 1);
#endregion
},
() =>
{
#region Tiers
List <AppDRESTTier> tiersList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTTier>(FilePathMap.APMTiersDataFilePath(jobTarget));
if (tiersList != null)
{
loggerConsole.Info("Extract Flowmaps for Tiers ({0} entities)", tiersList.Count);
int j = 0;
Parallel.ForEach(
tiersList,
parallelOptions,
() => 0,
(tier, loop, subtotal) =>
{
using (ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApiLocal.PrivateApiLogin();
if (File.Exists(FilePathMap.TierFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, tier)) == false)
{
string flowmapJson = controllerApiLocal.GetFlowmapTier(tier.id, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.TierFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, tier));
}
}
return(1);
}
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed {0} Tiers", tiersList.Count);
Interlocked.Add(ref numEntitiesTotal, tiersList.Count);
}
#endregion
},
() =>
{
#region Nodes
List <AppDRESTNode> nodesList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTNode>(FilePathMap.APMNodesDataFilePath(jobTarget));
if (nodesList != null)
{
loggerConsole.Info("Extract Flowmaps for Nodes ({0} entities)", nodesList.Count);
int j = 0;
Parallel.ForEach(
nodesList,
parallelOptions,
() => 0,
(node, loop, subtotal) =>
{
using (ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApiLocal.PrivateApiLogin();
if (File.Exists(FilePathMap.NodeFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, node)) == false)
{
string flowmapJson = controllerApiLocal.GetFlowmapNode(node.id, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.NodeFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, node));
}
}
return(1);
}
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed {0} Nodes", nodesList.Count);
Interlocked.Add(ref numEntitiesTotal, nodesList.Count);
}
#endregion
},
() =>
{
#region Backends
List <AppDRESTBackend> backendsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBackend>(FilePathMap.APMBackendsDataFilePath(jobTarget));
if (backendsList != null)
{
loggerConsole.Info("Extract Flowmaps for Backends ({0} entities)", backendsList.Count);
int j = 0;
Parallel.ForEach(
backendsList,
parallelOptions,
() => 0,
(backend, loop, subtotal) =>
{
ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
controllerApiLocal.PrivateApiLogin();
if (File.Exists(FilePathMap.BackendFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, backend)) == false)
{
string flowmapJson = controllerApiLocal.GetFlowmapBackend(backend.id, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.BackendFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, backend));
}
}
return(1);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed {0} Backends", backendsList.Count);
Interlocked.Add(ref numEntitiesTotal, backendsList.Count);
}
#endregion
},
() =>
{
#region Business Transactions
List <AppDRESTBusinessTransaction> businessTransactionsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBusinessTransaction>(FilePathMap.APMBusinessTransactionsDataFilePath(jobTarget));
if (businessTransactionsList != null)
{
loggerConsole.Info("Extract Flowmaps for Business Transactions ({0} entities)", businessTransactionsList.Count);
int j = 0;
Parallel.ForEach(
businessTransactionsList,
parallelOptions,
() => 0,
(businessTransaction, loop, subtotal) =>
{
using (ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApiLocal.PrivateApiLogin();
if (File.Exists(FilePathMap.BusinessTransactionFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, businessTransaction)) == false)
{
string flowmapJson = controllerApiLocal.GetFlowmapBusinessTransaction(jobTarget.ApplicationID, businessTransaction.id, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.BusinessTransactionFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, businessTransaction));
}
}
return(1);
}
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed {0} Business Transactions", businessTransactionsList.Count);
Interlocked.Add(ref numEntitiesTotal, businessTransactionsList.Count);
}
#endregion
}
);
stepTimingTarget.NumEntities = numEntitiesTotal;
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(programOptions, jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_SIM) == 0)
{
logger.Warn("No {0} targets to process", APPLICATION_TYPE_SIM);
loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_SIM);
return(true);
}
List <MetricExtractMapping> entityMetricExtractMappingList = getMetricsExtractMappingList(jobConfiguration);
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_SIM)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region Target step variables
stepTimingTarget.NumEntities = entityMetricExtractMappingList.Count;
#endregion
loggerConsole.Info("Extract Metrics for All Entities ({0} time ranges)", jobConfiguration.Input.HourlyTimeRanges.Count);
ParallelOptions parallelOptions = new ParallelOptions();
if (programOptions.ProcessSequentially == true)
{
parallelOptions.MaxDegreeOfParallelism = 1;
}
Parallel.Invoke(parallelOptions,
() =>
{
#region Machine
getMetricsForEntities(jobTarget, jobConfiguration, entityMetricExtractMappingList, SIMMachine.ENTITY_FOLDER, SIMMachine.ENTITY_TYPE);
#endregion
},
() =>
{
#region Network
getMetricsForEntities(jobTarget, jobConfiguration, entityMetricExtractMappingList, SIMMachineNetwork.ENTITY_FOLDER, SIMMachineNetwork.ENTITY_TYPE);
#endregion
}
);
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
