private int extractEvents(JobConfiguration jobConfiguration, JobTarget jobTarget, ControllerApi controllerApi, string eventType) { JArray listOfEvents = new JArray(); foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges) { long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.From); long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.To); string eventsJSON = controllerApi.GetEvents(jobTarget.ApplicationID, eventType, fromTimeUnix, toTimeUnix); if (eventsJSON != String.Empty) { // Load events JArray eventsInHour = JArray.Parse(eventsJSON); foreach (JObject interestingEvent in eventsInHour) { listOfEvents.Add(interestingEvent); } } } if (listOfEvents.Count > 0) { FileIOHelper.WriteJArrayToFile(listOfEvents, this.FilePathMap.EventsDataFilePath(jobTarget, eventType)); logger.Info("{0} {1} events from {2:o} to {3:o}", eventType, listOfEvents.Count, jobConfiguration.Input.TimeRange.From, jobConfiguration.Input.TimeRange.To); loggerConsole.Info("{0} {1} events", eventType, listOfEvents.Count); } return(listOfEvents.Count); }
internal HealthCheckRuleResult createHealthCheckRuleResult( JobTarget jobTarget, string entityType, string entityName, long entityID, HealthCheckRuleDescription hcrd) { HealthCheckRuleResult healthCheckRuleResult = new HealthCheckRuleResult(); healthCheckRuleResult.Controller = jobTarget.Controller; healthCheckRuleResult.Application = jobTarget.Application; healthCheckRuleResult.ApplicationID = jobTarget.ApplicationID; healthCheckRuleResult.EntityType = entityType; healthCheckRuleResult.EntityName = entityName; healthCheckRuleResult.EntityID = entityID; healthCheckRuleResult.Category = hcrd.Category; healthCheckRuleResult.Code = hcrd.Code; healthCheckRuleResult.Name = hcrd.Name; healthCheckRuleResult.Grade = 0; healthCheckRuleResult.EvaluationTime = DateTime.Now; return(healthCheckRuleResult); }
private void readGranularRangeOfMetricsIntoEntities( Dictionary <string, DBEntityBase> entitiesDictionaryByName, JobTimeRange jobTimeRange, JobTarget jobTarget, List <MetricExtractMapping> entityMetricExtractMappingList, string entityFolderName, string entityType, Dictionary <string, List <MetricValue> > metricValuesDictionary) { List <MetricExtractMapping> entityMetricExtractMappingListFiltered = entityMetricExtractMappingList.Where(m => m.EntityType == entityType).ToList(); foreach (MetricExtractMapping metricExtractMapping in entityMetricExtractMappingListFiltered) { List <AppDRESTMetric> metricData = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTMetric>(FilePathMap.MetricHourRangeDataFilePath(jobTarget, entityFolderName, metricExtractMapping.FolderName, jobTimeRange)); if (metricData != null) { List <MetricValue> metricValues = readMetricsIntoEntities(metricData, entitiesDictionaryByName, jobTarget, jobTimeRange); if (metricValues != null) { if (metricValuesDictionary.ContainsKey(metricExtractMapping.FolderName) == false) { metricValuesDictionary.Add(metricExtractMapping.FolderName, metricValues); } else { metricValuesDictionary[metricExtractMapping.FolderName].AddRange(metricValues); } } } } }
private int extractHealthRuleViolations(JobConfiguration jobConfiguration, JobTarget jobTarget, ControllerApi controllerApi) { JArray listOfHealthRuleViolations = new JArray(); foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges) { long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.From); long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.To); string healthRuleViolationsJSON = controllerApi.GetHealthRuleViolations(jobTarget.ApplicationID, fromTimeUnix, toTimeUnix); if (healthRuleViolationsJSON != String.Empty) { // Load health rule violations JArray healthRuleViolationsInHour = JArray.Parse(healthRuleViolationsJSON); foreach (JObject healthRuleViolation in healthRuleViolationsInHour) { listOfHealthRuleViolations.Add(healthRuleViolation); } } } if (listOfHealthRuleViolations.Count > 0) { FileIOHelper.WriteJArrayToFile(listOfHealthRuleViolations, FilePathMap.HealthRuleViolationsDataFilePath(jobTarget)); logger.Info("{0} health rule violations from {1:o} to {2:o}", listOfHealthRuleViolations.Count, jobConfiguration.Input.TimeRange.From, jobConfiguration.Input.TimeRange.To); loggerConsole.Info("{0} health rule violations", listOfHealthRuleViolations.Count); } return(listOfHealthRuleViolations.Count); }
private int extractAuditLogEvents(JobConfiguration jobConfiguration, JobTarget jobTarget, ControllerApi controllerApi) { JArray listOfAuditEvents = new JArray(); foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges) { string auditEventsJSON = controllerApi.GetAuditEvents(jobTimeRange.From, jobTimeRange.To); if (auditEventsJSON != String.Empty) { JArray listOfAuditEventsInTimeRange = JArray.Parse(auditEventsJSON); if (listOfAuditEventsInTimeRange != null) { // Load audit log events foreach (JObject auditEvent in listOfAuditEventsInTimeRange) { listOfAuditEvents.Add(auditEvent); } } } } if (listOfAuditEvents.Count > 0) { FileIOHelper.WriteJArrayToFile(listOfAuditEvents, FilePathMap.AuditEventsDataFilePath(jobTarget)); logger.Info("{0} audit events from {1:o} to {2:o}", listOfAuditEvents.Count, jobConfiguration.Input.TimeRange.From, jobConfiguration.Input.TimeRange.To); loggerConsole.Info("{0} audit events", listOfAuditEvents.Count); } return(listOfAuditEvents.Count); }
internal static JobTargetGroupData DeserializeJobTargetGroupData(JsonElement element) { ResourceIdentifier id = default; string name = default; ResourceType type = default; SystemData systemData = default; Optional <IList <JobTarget> > members = default; foreach (var property in element.EnumerateObject()) { if (property.NameEquals("id")) { id = new ResourceIdentifier(property.Value.GetString()); continue; } if (property.NameEquals("name")) { name = property.Value.GetString(); continue; } if (property.NameEquals("type")) { type = property.Value.GetString(); continue; } if (property.NameEquals("systemData")) { systemData = JsonSerializer.Deserialize <SystemData>(property.Value.ToString()); continue; } if (property.NameEquals("properties")) { if (property.Value.ValueKind == JsonValueKind.Null) { property.ThrowNonNullablePropertyIsNull(); continue; } foreach (var property0 in property.Value.EnumerateObject()) { if (property0.NameEquals("members")) { if (property0.Value.ValueKind == JsonValueKind.Null) { property0.ThrowNonNullablePropertyIsNull(); continue; } List <JobTarget> array = new List <JobTarget>(); foreach (var item in property0.Value.EnumerateArray()) { array.Add(JobTarget.DeserializeJobTarget(item)); } members = array; continue; } } continue; } } return(new JobTargetGroupData(id, name, type, systemData, Optional.ToList(members))); }
private HealthCheckRuleResult evaluate_Controller_Setting_Performance_Profile( HealthCheckRuleDescription hcrd, JobTarget jobTarget, Dictionary <string, HealthCheckSettingMapping> healthCheckSettingsDictionary, List <ControllerSetting> controllerSettingsList) { logger.Trace("Evaluating {0}", hcrd); HealthCheckRuleResult healthCheckRuleResult = createHealthCheckRuleResult(jobTarget, "Controller", jobTarget.Controller, 0, hcrd); healthCheckRuleResult.Application = "[ALL APPS]"; if (controllerSettingsList != null) { ControllerSetting controllerSettingProfile = controllerSettingsList.Where(s => s.Name == "performance.profile").FirstOrDefault(); if (controllerSettingProfile == null) { healthCheckRuleResult.Grade = 1; healthCheckRuleResult.Description = "Controller performance profile is unknown"; } else { healthCheckRuleResult.Description = String.Format("Controller performance profile is '{0}'", controllerSettingProfile.Value); // Defined in Enterprise Console via various // C:\AppDynamics\Platform\platform-admin\archives\controller\4.5.4.15417\playbooks\controller-<size>.groovy switch (controllerSettingProfile.Value.ToLower()) { case "internal": case "dev": case "demo": healthCheckRuleResult.Grade = 1; break; case "small": healthCheckRuleResult.Grade = 2; break; case "medium": healthCheckRuleResult.Grade = 3; break; case "large": healthCheckRuleResult.Grade = 4; break; case "extra-large": healthCheckRuleResult.Grade = 5; break; default: healthCheckRuleResult.Grade = 1; break; } } } return(healthCheckRuleResult); }
/* * 提供文本全局匹配“求职意向”,前缀使用精确匹配(意向行业),段内可以使用模糊匹配(行业,例如51job的简历) */ public JobTarget extractJobTarget(string resumeStr) { JobTarget jobTarget = new JobTarget(); // resumeFullStr="期望月薪 6001-8000元/月 目前状况: 应届毕业生"; // resumeFullStr = "期望月薪 :面议6001-8000元/月 目前状况: 应届毕业生"; // 行业 string industry = getIndustry(resumeStr, true); if (industry != null) { jobTarget.JobIndustry = industry; } // 职位 string jobCareer = getJobCareer(resumeStr, true); if (jobCareer != null) { jobTarget.JobCareer = jobCareer; } // 薪资 string salary = getSalary(resumeStr, true); if (salary != null) { jobTarget.Salary = salary; } // 地点 string location = getLocation(resumeStr, true); if (location != null) { jobTarget.JobLocation = location; } // 性质 string jobCatagory = getJobCatagory(resumeStr, true); if (jobCatagory != null) { jobTarget.JobCatagory = jobCatagory; } // 到岗时间 string enrollTime = getEnrollTime(resumeStr, true); if (enrollTime != null) { jobTarget.EnrollTime = enrollTime; } return(jobTarget); }
/* * 提供段内“求职意向”按行匹配结果 */ public JobTarget extractJobTarget(int start, int end) { JobTarget jobTarget = new JobTarget(); //获取段内所有数据 string line = getContent(start, end); // 行业 string industry = getIndustry(line, false); if (industry != null) { jobTarget.JobIndustry = industry; } // 职位 string jobCareer = getJobCareer(line, false); if (jobCareer != null) { jobTarget.JobCareer = jobCareer; } // 薪资 string salary = getSalary(line, false); if (salary != null) { jobTarget.Salary = salary; } // 地点 string location = getLocation(line, false); if (location != null) { jobTarget.JobLocation = location; } // 性质 string jobCatagory = getJobCatagory(line, false); if (jobCatagory != null) { jobTarget.JobCatagory = jobCatagory; } // 到岗时间 string enrollTime = getEnrollTime(line, false); if (enrollTime != null) { jobTarget.EnrollTime = enrollTime; } return(jobTarget); }
public JobTarget searchJobTarget() { // 从每个段中去搜索求职意向 JobTarget jobTarget = null; StringBuilder resumeTxtBuilder = new StringBuilder(); foreach (string line in resumeContentList) { resumeTxtBuilder.Append(line + " "); } jobTarget = extractJobTarget(resumeTxtBuilder.ToString()); return(jobTarget); }
/// <summary> /// Version of Controller should be reasonably latest /// </summary> /// <param name="jobTarget"></param> /// <param name="healthCheckSettingsDictionary"></param> /// <param name="controllerSummariesList"></param> /// <returns></returns> private HealthCheckRuleResult evaluate_Controller_Version( HealthCheckRuleDescription hcrd, JobTarget jobTarget, Dictionary <string, HealthCheckSettingMapping> healthCheckSettingsDictionary, List <ControllerSummary> controllerSummariesList) { logger.Trace("Evaluating {0}", hcrd); HealthCheckRuleResult healthCheckRuleResult = createHealthCheckRuleResult(jobTarget, "Controller", jobTarget.Controller, 0, hcrd); healthCheckRuleResult.Application = "[ALL APPS]"; if (controllerSummariesList != null && controllerSummariesList.Count > 0) { ControllerSummary controllerSummary = controllerSummariesList[0]; Version versionThisController = new Version(jobTarget.ControllerVersion); healthCheckRuleResult.Description = String.Format("The Controller version '{0}'", jobTarget.ControllerVersion, getVersionSetting(healthCheckSettingsDictionary, "ControllerVersionGrade5", "4.5")); if (versionThisController >= getVersionSetting(healthCheckSettingsDictionary, "ControllerVersionGrade5", "4.5")) { healthCheckRuleResult.Grade = 5; } else if (versionThisController >= getVersionSetting(healthCheckSettingsDictionary, "ControllerVersionGrade4", "4.4")) { healthCheckRuleResult.Grade = 4; } else if (versionThisController >= getVersionSetting(healthCheckSettingsDictionary, "ControllerVersionGrade3", "4.3")) { healthCheckRuleResult.Grade = 3; } else if (versionThisController >= getVersionSetting(healthCheckSettingsDictionary, "ControllerVersionGrade2", "4.2")) { healthCheckRuleResult.Grade = 2; } else { healthCheckRuleResult.Grade = 1; } } else { healthCheckRuleResult.Grade = 1; healthCheckRuleResult.Description = "No information about Controller version available"; } return(healthCheckRuleResult); }
private int extractEvents(JobConfiguration jobConfiguration, JobTarget jobTarget, ControllerApi controllerApi, string eventType) { loggerConsole.Info("Extract List of Events {0} ({1} time ranges)", eventType, jobConfiguration.Input.HourlyTimeRanges.Count); JArray listOfEventsArray = new JArray(); if (File.Exists(FilePathMap.ApplicationEventsDataFilePath(jobTarget, eventType)) == false) { foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges) { long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.From); long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.To); try { string eventsJSON = controllerApi.GetApplicationEvents(jobTarget.ApplicationID, eventType, fromTimeUnix, toTimeUnix); if (eventsJSON != String.Empty) { // Load events JArray eventsInHourArray = JArray.Parse(eventsJSON); foreach (JObject eventObject in eventsInHourArray) { listOfEventsArray.Add(eventObject); } Console.Write("[{0}]+", eventsInHourArray.Count); } } catch (Exception ex) { logger.Warn(ex); logger.Warn("Unable to parse JSON for Events Application={0}, EventType={1}, From={2}, To={3}", jobTarget.ApplicationID, eventType, fromTimeUnix, toTimeUnix); } } if (listOfEventsArray.Count > 0) { FileIOHelper.WriteJArrayToFile(listOfEventsArray, FilePathMap.ApplicationEventsDataFilePath(jobTarget, eventType)); logger.Info("{0} {1} events from {2:o} to {3:o}", eventType, listOfEventsArray.Count, jobConfiguration.Input.TimeRange.From, jobConfiguration.Input.TimeRange.To); loggerConsole.Info("{0} {1} events", eventType, listOfEventsArray.Count); } } return(listOfEventsArray.Count); }
public void extractJobTarget() { string seg_name = "career_objective"; if (segmentMap.ContainsKey(seg_name)) { TargetJobAnalyzer analyzer = new TargetJobAnalyzer(resumeContentList); SectionInfo sectionInfo = segmentMap[seg_name]; JobTarget jobTarget = analyzer.extractJobTarget(sectionInfo.Start, sectionInfo.End); resumedata.JobTarget = jobTarget; } else { TargetJobAnalyzer analyzer = new TargetJobAnalyzer(resumeContentList); JobTarget jobTarget = analyzer.searchJobTarget(); resumedata.JobTarget = jobTarget; } }
private int extractHealthRuleViolations(JobConfiguration jobConfiguration, JobTarget jobTarget, ControllerApi controllerApi) { JArray listOfHealthRuleViolations = new JArray(); if (File.Exists(FilePathMap.ApplicationHealthRuleViolationsDataFilePath(jobTarget)) == false) { foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges) { long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.From); long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.To); string healthRuleViolationsJSON = controllerApi.GetApplicationHealthRuleViolations(jobTarget.ApplicationID, fromTimeUnix, toTimeUnix); if (healthRuleViolationsJSON != String.Empty) { try { // Load health rule violations JArray healthRuleViolationsInHourArray = JArray.Parse(healthRuleViolationsJSON); foreach (JObject healthRuleViolationObject in healthRuleViolationsInHourArray) { listOfHealthRuleViolations.Add(healthRuleViolationObject); } } catch (Exception) { logger.Warn("Unable to parse JSON for HR Violations Application={0}, From={1}, To={2}", jobTarget.ApplicationID, fromTimeUnix, toTimeUnix); } } } if (listOfHealthRuleViolations.Count > 0) { FileIOHelper.WriteJArrayToFile(listOfHealthRuleViolations, FilePathMap.ApplicationHealthRuleViolationsDataFilePath(jobTarget)); logger.Info("{0} health rule violations from {1:o} to {2:o}", listOfHealthRuleViolations.Count, jobConfiguration.Input.TimeRange.From, jobConfiguration.Input.TimeRange.To); loggerConsole.Info("{0} health rule violations", listOfHealthRuleViolations.Count); } } return(listOfHealthRuleViolations.Count); }
private void extractListOfMetrics( JobTarget jobTarget, JobConfiguration jobConfiguration, string metricPathPrefix, string fileNamePrefix, int maxDepth) { using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { DateTime startTime = jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1].From; DateTime endTime = jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1].To; StringBuilder sbMetricPath = new StringBuilder(128); sbMetricPath.Append(metricPathPrefix); for (int currentDepth = 0; currentDepth <= maxDepth; currentDepth++) { sbMetricPath.Append("|*"); string metricPath = sbMetricPath.ToString(); loggerConsole.Trace("Depth {0:00}/{1:00}, {2}", currentDepth, maxDepth, metricPath); logger.Info("Retrieving metric lists for Application {0}({1}), Metric='{2}', From {3:o}, To {4:o}", jobTarget.Application, jobTarget.ApplicationID, metricPath, startTime, endTime); string metricsJson = String.Empty; string metricsDataFilePath = FilePathMap.MetricsListDataFilePath(jobTarget, fileNamePrefix, currentDepth); if (File.Exists(metricsDataFilePath) == false) { metricsJson = controllerApi.GetMetricData( jobTarget.ApplicationID, metricPath, UnixTimeHelper.ConvertToUnixTimestamp(startTime), UnixTimeHelper.ConvertToUnixTimestamp(endTime), true); FileIOHelper.SaveFileToPath(metricsJson, metricsDataFilePath); } } } }
/// <summary> /// Converts a Job Target model to an AzureSqlElasticJobTargetModel /// </summary> /// <param name="resourceGroupName">The resource group name</param> /// <param name="serverName">The server name</param> /// <param name="agentName">The agent name</param> /// <param name="targetGroupName">The target group name</param> /// <param name="target">The JobTarget model</param> /// <returns></returns> protected AzureSqlElasticJobTargetModel CreateAzureSqlElasticJobTargetModel( string resourceGroupName, string serverName, string agentName, string targetGroupName, JobTarget target) { return(new AzureSqlElasticJobTargetModel { ResourceGroupName = resourceGroupName, ServerName = serverName, AgentName = agentName, TargetGroupName = targetGroupName, MembershipType = target.MembershipType, TargetType = target.Type, RefreshCredentialName = new ResourceIdentifier(target.RefreshCredential).ResourceName, TargetServerName = target.ServerName, TargetDatabaseName = target.DatabaseName, TargetElasticPoolName = target.ElasticPoolName, TargetShardMapName = target.ShardMapName, }); }
/// <summary> /// We like SaaS controllers more than on premises /// </summary> /// <param name="jobTarget"></param> /// <param name="healthCheckSettingsDictionary"></param> /// <returns></returns> private HealthCheckRuleResult evaluate_Controller_SaaS_OnPrem( HealthCheckRuleDescription hcrd, JobTarget jobTarget, Dictionary <string, HealthCheckSettingMapping> healthCheckSettingsDictionary) { logger.Trace("Evaluating {0}", hcrd); HealthCheckRuleResult healthCheckRuleResult = createHealthCheckRuleResult(jobTarget, "Controller", jobTarget.Controller, 0, hcrd); healthCheckRuleResult.Application = "[ALL APPS]"; if (jobTarget.Controller.ToLower().Contains("saas.appdynamics") == true) { healthCheckRuleResult.Grade = 5; healthCheckRuleResult.Description = "Controller is running in AppDynamics SaaS cloud"; } else { healthCheckRuleResult.Grade = 3; healthCheckRuleResult.Description = "Controller is running in OnPremises configuration"; } return(healthCheckRuleResult); }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(jobConfiguration) == false) { return(true); } if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_SIM) == 0) { return(true); } // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_SIM) { continue; } StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; stepTimingTarget.NumEntities = 1; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); // Set up controller access using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { #region Prepare time range long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From); long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To); long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000); #endregion #region Tiers loggerConsole.Info("List of Tiers"); string tiersJSON = controllerApi.GetSIMListOfTiers(); if (tiersJSON != String.Empty) { FileIOHelper.SaveFileToPath(tiersJSON, FilePathMap.SIMTiersDataFilePath(jobTarget)); } #endregion #region Nodes loggerConsole.Info("List of Nodes"); string nodesJSON = controllerApi.GetSIMListOfNodes(); if (nodesJSON != String.Empty) { FileIOHelper.SaveFileToPath(nodesJSON, FilePathMap.SIMNodesDataFilePath(jobTarget)); } #endregion #region Groups loggerConsole.Info("List of Groups"); string groupsJSON = controllerApi.GetSIMListOfGroups(); if (groupsJSON != String.Empty) { FileIOHelper.SaveFileToPath(groupsJSON, FilePathMap.SIMGroupsDataFilePath(jobTarget)); } #endregion #region Service Availability loggerConsole.Info("List of Service Availability"); string sasJSON = controllerApi.GetSIMListOfServiceAvailability(); if (sasJSON != String.Empty) { FileIOHelper.SaveFileToPath(sasJSON, FilePathMap.SIMServiceAvailabilitiesDataFilePath(jobTarget)); } JArray sasList = FileIOHelper.LoadJArrayFromFile(FilePathMap.SIMServiceAvailabilitiesDataFilePath(jobTarget)); if (sasList != null) { loggerConsole.Info("Service Availability Details ({0} entities)", sasList.Count); foreach (JToken saToken in sasList) { string saJSON = controllerApi.GetSIMServiceAvailability((long)saToken["id"]); if (saJSON != String.Empty) { FileIOHelper.SaveFileToPath(saJSON, FilePathMap.SIMServiceAvailabilityDataFilePath(jobTarget, saToken["name"].ToString(), (long)saToken["id"])); } string saEventsJSON = controllerApi.GetSIMServiceAvailabilityEvents((long)saToken["id"], fromTimeUnix, toTimeUnix, differenceInMinutes); if (saEventsJSON != String.Empty && saEventsJSON != "[]") { FileIOHelper.SaveFileToPath(saEventsJSON, FilePathMap.SIMServiceAvailabilityEventsDataFilePath(jobTarget, saToken["name"].ToString(), (long)saToken["id"], jobConfiguration.Input.TimeRange)); } } loggerConsole.Info("Completed {0} Service Availabilities", sasList.Count); } #endregion #region Machines loggerConsole.Info("List of Machines"); string machinesJSON = controllerApi.GetSIMListOfMachines(); if (machinesJSON != String.Empty) { FileIOHelper.SaveFileToPath(machinesJSON, FilePathMap.SIMMachinesDataFilePath(jobTarget)); } JArray machinesArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.SIMMachinesDataFilePath(jobTarget)); if (machinesArray != null) { loggerConsole.Info("Machine, Container Details and Processes ({0} entities)", machinesArray.Count); int j = 0; var listOfMachinesChunks = machinesArray.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_MACHINES_TO_PROCESS_PER_THREAD); Parallel.ForEach <List <JToken>, int>( listOfMachinesChunks, new ParallelOptions { MaxDegreeOfParallelism = MACHINES_EXTRACT_NUMBER_OF_THREADS }, () => 0, (listOfMachinesChunk, loop, subtotal) => { // Set up controller access using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { foreach (JToken machineToken in listOfMachinesChunk) { if (File.Exists(FilePathMap.SIMMachineDataFilePath(jobTarget, machineToken["name"].ToString(), (long)machineToken["id"])) == false) { string machineJSON = controllerApi.GetSIMMachine((long)machineToken["id"]); if (machineJSON != String.Empty) { FileIOHelper.SaveFileToPath(machineJSON, FilePathMap.SIMMachineDataFilePath(jobTarget, machineToken["name"].ToString(), (long)machineToken["id"])); } string machineDockerJSON = controllerApi.GetSIMMachineDockerContainers((long)machineToken["id"]); if (machineDockerJSON != String.Empty && machineDockerJSON != "[]") { FileIOHelper.SaveFileToPath(machineDockerJSON, FilePathMap.SIMMachineDockerContainersDataFilePath(jobTarget, machineToken["name"].ToString(), (long)machineToken["id"])); } string machineProcessesJSON = controllerApi.GetSIMMachineProcesses((long)machineToken["id"], fromTimeUnix, toTimeUnix, differenceInMinutes); if (machineProcessesJSON != String.Empty && machineProcessesJSON != "[]") { FileIOHelper.SaveFileToPath(machineProcessesJSON, FilePathMap.SIMMachineProcessesDataFilePath(jobTarget, machineToken["name"].ToString(), (long)machineToken["id"], jobConfiguration.Input.TimeRange)); } } } return(listOfMachinesChunk.Count); } }, (finalResult) => { Interlocked.Add(ref j, finalResult); Console.Write("[{0}].", j); } ); loggerConsole.Info("Completed {0} Machines", machinesArray.Count); } #endregion } } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(jobConfiguration) == false) { return(true); } bool haveProcessedAtLeastOneDBCollector = false; // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); if (jobTarget.Type == APPLICATION_TYPE_DB) { if (haveProcessedAtLeastOneDBCollector == false) { haveProcessedAtLeastOneDBCollector = true; } else { continue; } } int numEventsTotal = 0; #region Health Rule violations using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { loggerConsole.Info("Extract List of Health Rule Violations ({0} time ranges)", jobConfiguration.Input.HourlyTimeRanges.Count); numEventsTotal = numEventsTotal + extractHealthRuleViolations(jobConfiguration, jobTarget, controllerApi); } #endregion #region Events loggerConsole.Info("Extract {0} event types ({1} time ranges)", EVENT_TYPES.Count, jobConfiguration.Input.HourlyTimeRanges.Count); Parallel.ForEach( EVENT_TYPES, new ParallelOptions { MaxDegreeOfParallelism = EVENTS_EXTRACT_NUMBER_OF_THREADS }, () => 0, (eventType, loop, subtotal) => { using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { int numEventsInType = extractEvents(jobConfiguration, jobTarget, controllerApiParallel, eventType); subtotal = subtotal + numEventsInType; return(subtotal); } }, (finalResult) => { Interlocked.Add(ref numEventsTotal, finalResult); } ); loggerConsole.Info("{0} events total", numEventsTotal); #endregion stepTimingTarget.NumEntities = numEventsTotal; } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(programOptions, jobConfiguration) == false) { return(true); } if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_APM) == 0) { logger.Warn("No {0} targets to process", APPLICATION_TYPE_APM); loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_APM); return(true); } // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_APM) { continue; } StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; stepTimingTarget.NumEntities = 9; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Prepare time range long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1].From); long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1].To); long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000); #endregion // Set up controller access using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { #region Application loggerConsole.Info("Application Name"); string applicationJSON = controllerApi.GetAPMApplication(jobTarget.ApplicationID); if (applicationJSON != String.Empty) { FileIOHelper.SaveFileToPath(applicationJSON, FilePathMap.APMApplicationDataFilePath(jobTarget)); } #endregion #region Tiers loggerConsole.Info("List of Tiers"); string tiersJSON = controllerApi.GetAPMTiers(jobTarget.ApplicationID); if (tiersJSON != String.Empty) { FileIOHelper.SaveFileToPath(tiersJSON, FilePathMap.APMTiersDataFilePath(jobTarget)); } #endregion #region Nodes loggerConsole.Info("List of Nodes"); string nodesJSON = controllerApi.GetAPMNodes(jobTarget.ApplicationID); if (nodesJSON != String.Empty) { FileIOHelper.SaveFileToPath(nodesJSON, FilePathMap.APMNodesDataFilePath(jobTarget)); } #endregion #region Backends loggerConsole.Info("List of Backends"); string backendsJSON = controllerApi.GetAPMBackends(jobTarget.ApplicationID); if (backendsJSON != String.Empty) { FileIOHelper.SaveFileToPath(backendsJSON, FilePathMap.APMBackendsDataFilePath(jobTarget)); } controllerApi.PrivateApiLogin(); backendsJSON = controllerApi.GetAPMBackendsAdditionalDetail(jobTarget.ApplicationID); if (backendsJSON != String.Empty) { FileIOHelper.SaveFileToPath(backendsJSON, FilePathMap.APMBackendsDetailDataFilePath(jobTarget)); } List <AppDRESTBackend> backendsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBackend>(FilePathMap.APMBackendsDataFilePath(jobTarget)); if (backendsList != null) { loggerConsole.Info("DBMon Mappings for Backends ({0} entities)", backendsList.Count); int j = 0; var listOfBackendsInHourChunks = backendsList.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_BACKENDS_TO_PROCESS_PER_THREAD); ParallelOptions parallelOptions = new ParallelOptions(); if (programOptions.ProcessSequentially == true) { parallelOptions.MaxDegreeOfParallelism = 1; } else { parallelOptions.MaxDegreeOfParallelism = BACKEND_PROPERTIES_EXTRACT_NUMBER_OF_THREADS; } Parallel.ForEach <List <AppDRESTBackend>, int>( listOfBackendsInHourChunks, parallelOptions, () => 0, (listOfBackendsInHourChunk, loop, subtotal) => { // Set up controller access ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)); // Login into private API controllerApiParallel.PrivateApiLogin(); foreach (AppDRESTBackend backend in listOfBackendsInHourChunk) { if (File.Exists(FilePathMap.APMBackendToDBMonMappingDataFilePath(jobTarget, backend)) == false) { string backendToDBMonMappingJSON = controllerApi.GetAPMBackendToDBMonMapping(backend.id); if (backendToDBMonMappingJSON != String.Empty) { FileIOHelper.SaveFileToPath(backendToDBMonMappingJSON, FilePathMap.APMBackendToDBMonMappingDataFilePath(jobTarget, backend)); } } } return(listOfBackendsInHourChunk.Count); }, (finalResult) => { Interlocked.Add(ref j, finalResult); Console.Write("[{0}].", j); } ); loggerConsole.Info("Completed {0} Backends", backendsList.Count); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + backendsList.Count; } #endregion #region Business Transactions loggerConsole.Info("List of Business Transactions"); string businessTransactionsJSON = controllerApi.GetAPMBusinessTransactions(jobTarget.ApplicationID); if (businessTransactionsJSON != String.Empty) { FileIOHelper.SaveFileToPath(businessTransactionsJSON, FilePathMap.APMBusinessTransactionsDataFilePath(jobTarget)); } #endregion #region Service Endpoints loggerConsole.Info("List of Service Endpoints"); string serviceEndPointsJSON = controllerApi.GetAPMServiceEndpoints(jobTarget.ApplicationID); if (serviceEndPointsJSON != String.Empty) { FileIOHelper.SaveFileToPath(serviceEndPointsJSON, FilePathMap.APMServiceEndpointsDataFilePath(jobTarget)); } controllerApi.PrivateApiLogin(); serviceEndPointsJSON = controllerApi.GetAPMServiceEndpointsAdditionalDetail(jobTarget.ApplicationID); if (serviceEndPointsJSON != String.Empty) { FileIOHelper.SaveFileToPath(serviceEndPointsJSON, FilePathMap.APMServiceEndpointsDetailDataFilePath(jobTarget)); } #endregion #region Errors loggerConsole.Info("List of Errors"); string errorsJSON = controllerApi.GetAPMErrors(jobTarget.ApplicationID); if (errorsJSON != String.Empty) { FileIOHelper.SaveFileToPath(errorsJSON, FilePathMap.APMErrorsDataFilePath(jobTarget)); } #endregion #region Information Points loggerConsole.Info("List of Information Points"); string informationPointsJSON = controllerApi.GetAPMInformationPoints(jobTarget.ApplicationID); if (informationPointsJSON != String.Empty) { FileIOHelper.SaveFileToPath(informationPointsJSON, FilePathMap.APMInformationPointsDataFilePath(jobTarget)); } controllerApi.PrivateApiLogin(); string informationPointsDetailJSON = controllerApi.GetAPMInformationPointsAdditionalDetail(jobTarget.ApplicationID); if (informationPointsDetailJSON != String.Empty) { FileIOHelper.SaveFileToPath(informationPointsDetailJSON, FilePathMap.APMInformationPointsDetailDataFilePath(jobTarget)); } #endregion #region Node Properties List <AppDRESTNode> nodesList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTNode>(FilePathMap.APMNodesDataFilePath(jobTarget)); if (nodesList != null) { loggerConsole.Info("Node Properties for Nodes ({0} entities)", nodesList.Count); int j = 0; var listOfNodesInHourChunks = nodesList.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_NODES_TO_PROCESS_PER_THREAD); ParallelOptions parallelOptions = new ParallelOptions(); if (programOptions.ProcessSequentially == true) { parallelOptions.MaxDegreeOfParallelism = 1; } else { parallelOptions.MaxDegreeOfParallelism = NODE_PROPERTIES_EXTRACT_NUMBER_OF_THREADS; } Parallel.ForEach <List <AppDRESTNode>, int>( listOfNodesInHourChunks, parallelOptions, () => 0, (listOfNodesInHourChunk, loop, subtotal) => { // Set up controller access ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)); // Login into private API controllerApiParallel.PrivateApiLogin(); foreach (AppDRESTNode node in listOfNodesInHourChunk) { if (File.Exists(FilePathMap.APMNodeRuntimePropertiesDataFilePath(jobTarget, node)) == false) { string nodePropertiesJSON = controllerApi.GetAPMNodeProperties(node.id); if (nodePropertiesJSON != String.Empty) { FileIOHelper.SaveFileToPath(nodePropertiesJSON, FilePathMap.APMNodeRuntimePropertiesDataFilePath(jobTarget, node)); } } if (File.Exists(FilePathMap.APMNodeMetadataDataFilePath(jobTarget, node)) == false) { string nodeMetadataJSON = controllerApi.GetAPMNodeMetadata(jobTarget.ApplicationID, node.id); if (nodeMetadataJSON != String.Empty) { FileIOHelper.SaveFileToPath(nodeMetadataJSON, FilePathMap.APMNodeMetadataDataFilePath(jobTarget, node)); } } } return(listOfNodesInHourChunk.Count); }, (finalResult) => { Interlocked.Add(ref j, finalResult); Console.Write("[{0}].", j); } ); loggerConsole.Info("Completed {0} Nodes", nodesList.Count); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + nodesList.Count; } #endregion #region Backend to Tier Mappings List <AppDRESTTier> tiersRESTList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTTier>(FilePathMap.APMTiersDataFilePath(jobTarget)); if (tiersRESTList != null) { loggerConsole.Info("Backend to Tier Mappings ({0} entities)", tiersRESTList.Count); int j = 0; foreach (AppDRESTTier tier in tiersRESTList) { string backendMappingsJSON = controllerApi.GetAPMBackendToTierMapping(tier.id); if (backendMappingsJSON != String.Empty && backendMappingsJSON != "[ ]") { FileIOHelper.SaveFileToPath(backendMappingsJSON, FilePathMap.APMBackendToTierMappingDataFilePath(jobTarget, tier)); } if (j % 10 == 0) { Console.Write("[{0}].", j); } j++; } loggerConsole.Info("Completed {0} Tiers", tiersRESTList.Count); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + tiersRESTList.Count; } #endregion #region Overflow Business Transactions if (tiersRESTList != null) { loggerConsole.Info("Contents of Overflow Business Transaction in Tiers ({0} entities)", tiersRESTList.Count); int j = 0; foreach (AppDRESTTier tier in tiersRESTList) { JArray droppedBTsArray = new JArray(); JArray droppedBTsDebugModeArray = new JArray(); bool noMoreBTs = false; long currentFetchedEventCount = 0; long endEventID = 0; while (noMoreBTs == false) { string batchOfBTsJSON = controllerApi.GetAPMBusinessTransactionsInOverflow(tier.id, currentFetchedEventCount, endEventID, fromTimeUnix, toTimeUnix, differenceInMinutes); if (batchOfBTsJSON != String.Empty) { JObject batchOfBTsContainer = JObject.Parse(batchOfBTsJSON); if (batchOfBTsContainer != null) { // Copy out both of the containers, not sure why there are multiple if (isTokenPropertyNull(batchOfBTsContainer, "droppedTransactionItemList") == false) { foreach (JObject btObject in batchOfBTsContainer["droppedTransactionItemList"]) { droppedBTsArray.Add(btObject); } } if (isTokenPropertyNull(batchOfBTsContainer, "debugModeDroppedTransactionItemList") == false) { foreach (JObject btObject in batchOfBTsContainer["debugModeDroppedTransactionItemList"]) { droppedBTsDebugModeArray.Add(btObject); } } currentFetchedEventCount = getLongValueFromJToken(batchOfBTsContainer, "eventSummariesCount"); endEventID = getLongValueFromJToken(batchOfBTsContainer, "endEventId"); if (currentFetchedEventCount == 0 || endEventID == 0) { // Done getting batches noMoreBTs = true; } } } else { noMoreBTs = true; } } if (droppedBTsArray.Count > 0) { FileIOHelper.SaveFileToPath(droppedBTsArray.ToString(), FilePathMap.APMTierOverflowBusinessTransactionRegularDataFilePath(jobTarget, tier)); } if (droppedBTsDebugModeArray.Count > 0) { FileIOHelper.SaveFileToPath(droppedBTsDebugModeArray.ToString(), FilePathMap.APMTierOverflowBusinessTransactionDebugDataFilePath(jobTarget, tier)); } if (j % 10 == 0) { Console.Write("[{0}].", j); } j++; } loggerConsole.Info("Completed {0} Tiers", tiersRESTList.Count); } #endregion } } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public void DisplayJobTargetEndedStatus(JobConfiguration jobConfiguration, JobTarget jobTarget, int jobTargetIndex, Stopwatch stopWatch) { logger.Info("{0}({0:d}): [{1}/{2}], {3} {4} duration {5:c} ({6} ms)", jobConfiguration.Status, jobTargetIndex, jobConfiguration.Target.Count, jobTarget.Controller, jobTarget.Application, stopWatch.Elapsed, stopWatch.ElapsedMilliseconds); loggerConsole.Trace("{0}({0:d}): [{1}/{2}], {3} {4} duration {5:c} ({6} ms)", jobConfiguration.Status, jobTargetIndex, jobConfiguration.Target.Count, jobTarget.Controller, jobTarget.Application, stopWatch.Elapsed, stopWatch.ElapsedMilliseconds); }
public void DisplayJobTargetStartingStatus(JobConfiguration jobConfiguration, JobTarget jobTarget, int jobTargetIndex) { logger.Info("{0}({0:d}): [{1}/{2}], {3} {4}({5}) [{6}]", jobConfiguration.Status, jobTargetIndex, jobConfiguration.Target.Count, jobTarget.Controller, jobTarget.Application, jobTarget.ApplicationID, jobTarget.Type); loggerConsole.Info("{0}({0:d}): [{1}/{2}], {3} {4}({5}) [{6}]", jobConfiguration.Status, jobTargetIndex, jobConfiguration.Target.Count, jobTarget.Controller, jobTarget.Application, jobTarget.ApplicationID, jobTarget.Type); }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(jobConfiguration) == false) { return(true); } bool reportFolderCleaned = false; // Process each Controller once int i = 0; var controllers = jobConfiguration.Target.GroupBy(t => t.Controller); foreach (var controllerGroup in controllers) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = controllerGroup.ToList()[0]; StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; stepTimingTarget.NumEntities = 0; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Controller Settings loggerConsole.Info("Controller Settings"); List <ControllerSetting> controllerSettingsList = new List <ControllerSetting>(); JArray controllerSettingsArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.ControllerSettingsDataFilePath(jobTarget)); if (controllerSettingsArray != null) { foreach (JObject controllerSettingObject in controllerSettingsArray) { ControllerSetting controllerSetting = new ControllerSetting(); controllerSetting.Controller = jobTarget.Controller; controllerSetting.ControllerLink = String.Format(DEEPLINK_CONTROLLER, controllerSetting.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); controllerSetting.Name = getStringValueFromJToken(controllerSettingObject, "name"); controllerSetting.Description = getStringValueFromJToken(controllerSettingObject, "description"); controllerSetting.Value = getStringValueFromJToken(controllerSettingObject, "value"); controllerSetting.Updateable = getBoolValueFromJToken(controllerSettingObject, "updateable"); controllerSetting.Scope = getStringValueFromJToken(controllerSettingObject, "scope"); controllerSettingsList.Add(controllerSetting); } } controllerSettingsList = controllerSettingsList.OrderBy(c => c.Name).ToList(); FileIOHelper.WriteListToCSVFile(controllerSettingsList, new ControllerSettingReportMap(), FilePathMap.ControllerSettingsIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + controllerSettingsList.Count; #endregion #region HTTP Templates loggerConsole.Info("HTTP Templates"); List <HTTPAlertTemplate> httpTemplatesList = new List <HTTPAlertTemplate>(); JArray httpTemplatesArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.HTTPTemplatesDataFilePath(jobTarget)); JArray httpTemplatesDetailArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.HTTPTemplatesDetailDataFilePath(jobTarget)); if (httpTemplatesArray != null) { foreach (JObject httpTemplateObject in httpTemplatesArray) { HTTPAlertTemplate httpAlertTemplate = new HTTPAlertTemplate(); httpAlertTemplate.Controller = jobTarget.Controller; httpAlertTemplate.Name = getStringValueFromJToken(httpTemplateObject, "name"); httpAlertTemplate.Method = getStringValueFromJToken(httpTemplateObject, "method"); httpAlertTemplate.Scheme = getStringValueFromJToken(httpTemplateObject, "scheme"); httpAlertTemplate.Host = getStringValueFromJToken(httpTemplateObject, "host"); httpAlertTemplate.Port = getIntValueFromJToken(httpTemplateObject, "port"); httpAlertTemplate.Path = getStringValueFromJToken(httpTemplateObject, "path"); httpAlertTemplate.Query = getStringValueFromJToken(httpTemplateObject, "query"); httpAlertTemplate.AuthType = getStringValueFromJToken(httpTemplateObject, "authType"); httpAlertTemplate.AuthUsername = getStringValueFromJToken(httpTemplateObject, "authUsername"); httpAlertTemplate.AuthPassword = getStringValueFromJToken(httpTemplateObject, "authPassword"); httpAlertTemplate.Headers = getStringValueOfObjectFromJToken(httpTemplateObject, "headers", true); if (isTokenPropertyNull(httpTemplateObject, "payloadTemplate") == false) { httpAlertTemplate.ContentType = getStringValueFromJToken(httpTemplateObject["payloadTemplate"], "httpRequestActionMediaType"); httpAlertTemplate.FormData = getStringValueOfObjectFromJToken(httpTemplateObject["payloadTemplate"], "formDataPairs", true); httpAlertTemplate.Payload = getStringValueFromJToken(httpTemplateObject["payloadTemplate"], "payload"); } httpAlertTemplate.ConnectTimeout = getLongValueFromJToken(httpTemplateObject, "connectTimeoutInMillis"); httpAlertTemplate.SocketTimeout = getLongValueFromJToken(httpTemplateObject, "socketTimeoutInMillis"); httpAlertTemplate.ResponseAny = getStringValueOfObjectFromJToken(httpTemplateObject, "responseMatchCriteriaAnyTemplate"); httpAlertTemplate.ResponseNone = getStringValueOfObjectFromJToken(httpTemplateObject, "responseMatchCriteriaNoneTemplate"); if (httpTemplatesDetailArray != null) { JToken httpAlertTemplateToken = httpTemplatesDetailArray.Where(t => t["name"].ToString() == httpAlertTemplate.Name).FirstOrDefault(); if (httpAlertTemplateToken != null) { httpAlertTemplate.TemplateID = getLongValueFromJToken(httpAlertTemplateToken, "id"); } } httpTemplatesList.Add(httpAlertTemplate); } } httpTemplatesList = httpTemplatesList.OrderBy(c => c.Name).ToList(); FileIOHelper.WriteListToCSVFile(httpTemplatesList, new HTTPAlertTemplateReportMap(), FilePathMap.HTTPTemplatesIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + httpTemplatesList.Count; #endregion #region Email Templates loggerConsole.Info("Email Templates"); List <EmailAlertTemplate> emailTemplatesList = new List <EmailAlertTemplate>(); JArray emailTemplatesArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.EmailTemplatesDataFilePath(jobTarget)); JArray emailTemplatesDetailArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.EmailTemplatesDetailDataFilePath(jobTarget)); if (emailTemplatesArray != null) { foreach (JObject emailTemplateObject in emailTemplatesArray) { EmailAlertTemplate emailAlertTemplate = new EmailAlertTemplate(); emailAlertTemplate.Controller = jobTarget.Controller; emailAlertTemplate.Name = getStringValueFromJToken(emailTemplateObject, "name"); emailAlertTemplate.OneEmailPerEvent = getBoolValueFromJToken(emailTemplateObject, "oneEmailPerEvent"); emailAlertTemplate.EventLimit = getLongValueFromJToken(emailTemplateObject, "eventClampLimit"); try { string[] emails = emailTemplateObject["toRecipients"].Select(s => getStringValueFromJToken(s, "value")).ToArray(); emailAlertTemplate.To = String.Join(";", emails); } catch { } try { string[] emails = emailTemplateObject["ccRecipients"].Select(s => getStringValueFromJToken(s, "value")).ToArray(); emailAlertTemplate.CC = String.Join(";", emails); } catch { } try { string[] emails = emailTemplateObject["bccRecipients"].Select(s => getStringValueFromJToken(s, "value")).ToArray(); emailAlertTemplate.BCC = String.Join(";", emails); } catch { } try { string[] emails = emailTemplateObject["testToRecipients"].Select(s => getStringValueFromJToken(s, "value")).ToArray(); emailAlertTemplate.TestTo = String.Join(";", emails); } catch { } try { string[] emails = emailTemplateObject["testCcRecipients"].Select(s => getStringValueFromJToken(s, "value")).ToArray(); emailAlertTemplate.TestCC = String.Join(";", emails); } catch { } try { string[] emails = emailTemplateObject["testBccRecipients"].Select(s => getStringValueFromJToken(s, "value")).ToArray(); emailAlertTemplate.TestBCC = String.Join(";", emails); } catch { } emailAlertTemplate.TestLogLevel = getStringValueFromJToken(emailTemplateObject, "testLogLevel"); emailAlertTemplate.Headers = getStringValueOfObjectFromJToken(emailTemplateObject, "headers", true); emailAlertTemplate.Subject = getStringValueFromJToken(emailTemplateObject, "subject"); emailAlertTemplate.TextBody = getStringValueFromJToken(emailTemplateObject, "textBody"); emailAlertTemplate.HTMLBody = getStringValueFromJToken(emailTemplateObject, "htmlBody"); emailAlertTemplate.IncludeHTMLBody = getBoolValueFromJToken(emailTemplateObject, "includeHtmlBody"); emailAlertTemplate.Properties = getStringValueOfObjectFromJToken(emailTemplateObject, "defaultCustomProperties"); emailAlertTemplate.TestProperties = getStringValueOfObjectFromJToken(emailTemplateObject, "testPropertiesPairs"); emailAlertTemplate.EventTypes = getStringValueOfObjectFromJToken(emailTemplateObject, "eventTypeCountPairs"); if (emailTemplatesDetailArray != null) { JToken emailTemplateDetailToken = emailTemplatesDetailArray.Where(t => t["name"].ToString() == emailAlertTemplate.Name).FirstOrDefault(); if (emailTemplateDetailToken != null) { emailAlertTemplate.TemplateID = getLongValueFromJToken(emailTemplateDetailToken, "id"); } } emailTemplatesList.Add(emailAlertTemplate); } } emailTemplatesList = emailTemplatesList.OrderBy(c => c.Name).ToList(); FileIOHelper.WriteListToCSVFile(emailTemplatesList, new EmailAlertTemplateReportMap(), FilePathMap.EmailTemplatesIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + emailTemplatesList.Count; #endregion #region Combine All for Report CSV // If it is the first one, clear out the combined folder if (reportFolderCleaned == false) { FileIOHelper.DeleteFolder(FilePathMap.ControllerSettingsReportFolderPath()); Thread.Sleep(1000); FileIOHelper.CreateFolder(FilePathMap.ControllerSettingsReportFolderPath()); reportFolderCleaned = true; } // Append all the individual report files into one if (File.Exists(FilePathMap.ControllerSettingsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.ControllerSettingsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.ControllerSettingsReportFilePath(), FilePathMap.ControllerSettingsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.HTTPTemplatesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.HTTPTemplatesIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.HTTPTemplatesReportFilePath(), FilePathMap.HTTPTemplatesIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.EmailTemplatesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.EmailTemplatesIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.EmailTemplatesReportFilePath(), FilePathMap.EmailTemplatesIndexFilePath(jobTarget)); } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } i++; } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
private static MOBILENetworkRequestRule fillNetworkRequestRule(JObject ruleObject, JobTarget jobTarget) { MOBILENetworkRequestRule webPageDetectionRule = new MOBILENetworkRequestRule(); webPageDetectionRule.Controller = jobTarget.Controller; webPageDetectionRule.ControllerLink = String.Format(DEEPLINK_CONTROLLER, jobTarget.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); webPageDetectionRule.ApplicationName = jobTarget.Application; webPageDetectionRule.ApplicationID = jobTarget.ApplicationID; webPageDetectionRule.ApplicationLink = String.Format(DEEPLINK_MOBILE_APPLICATION, jobTarget.Controller, jobTarget.ParentApplicationID, jobTarget.ApplicationID, DEEPLINK_TIMERANGE_LAST_15_MINUTES); webPageDetectionRule.RuleName = getStringValueFromJToken(ruleObject, "name"); webPageDetectionRule.IsEnabled = getBoolValueFromJToken(ruleObject, "enabled"); webPageDetectionRule.IsDefault = getBoolValueFromJToken(ruleObject, "isDefault"); if (webPageDetectionRule.IsDefault && webPageDetectionRule.RuleName.Length == 0) { webPageDetectionRule.RuleName = "[DEFAULT]"; } webPageDetectionRule.Priority = getIntValueFromJToken(ruleObject, "priority"); webPageDetectionRule.MatchMobileApp = getMatchRuleDescription((JToken)ruleObject["matchOnMobileApplicationName"]); webPageDetectionRule.MatchURL = getMatchRuleDescription((JToken)ruleObject["matchOnURL"]); webPageDetectionRule.MatchIPAddress = getMatchRuleDescription((JToken)ruleObject["matchOnIpAddressMasks"]); webPageDetectionRule.MatchUserAgent = getMatchRuleDescription((JToken)ruleObject["matchOnUserAgent"]); webPageDetectionRule.MatchUserAgentType = getMatchRuleDescription((JToken)ruleObject["matchOnUserAgentType"]); if (isTokenPropertyNull(ruleObject, "pageNamingConfig") == false) { JObject pageNamingConfigObject = (JObject)ruleObject["pageNamingConfig"]; webPageDetectionRule.UseProtocol = getBoolValueFromJToken(pageNamingConfigObject, "useProtocol"); webPageDetectionRule.UseDomain = getBoolValueFromJToken(pageNamingConfigObject, "useDomainName"); webPageDetectionRule.UseURL = getBoolValueFromJToken(pageNamingConfigObject, "useURL"); webPageDetectionRule.UseRegex = getBoolValueFromJToken(pageNamingConfigObject, "useRegex"); webPageDetectionRule.UseHTTP = getBoolValueFromJToken(pageNamingConfigObject, "useHttpMethod"); webPageDetectionRule.NamingType = getStringValueFromJToken(pageNamingConfigObject, "type"); webPageDetectionRule.AnchorType = getStringValueFromJToken(pageNamingConfigObject, "anchorType"); webPageDetectionRule.UrlSegments = getStringValueOfObjectFromJToken(pageNamingConfigObject, "urlMatchSegments", true); webPageDetectionRule.AnchorSegments = getStringValueOfObjectFromJToken(pageNamingConfigObject, "anchorMatchSegments", true); webPageDetectionRule.RegexGroups = getStringValueOfObjectFromJToken(pageNamingConfigObject, "regexGroupConfig", true); webPageDetectionRule.QueryStrings = getStringValueOfObjectFromJToken(pageNamingConfigObject, "queryStringMatch", true).Replace("[]", ""); webPageDetectionRule.DomainNameType = getStringValueFromJToken(pageNamingConfigObject, "domainNameType"); } return(webPageDetectionRule); }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(programOptions, jobConfiguration) == false) { return(true); } if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_MOBILE) == 0) { logger.Warn("No {0} targets to process", APPLICATION_TYPE_MOBILE); loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_MOBILE); return(true); } #region Template comparisons if (jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE.Controller == BLANK_APPLICATION_CONTROLLER && jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE.Application == BLANK_APPLICATION_MOBILE) { jobConfiguration.Target.Add(jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE); } else { // Check if there is a valid reference application JobTarget jobTargetReferenceApp = jobConfiguration.Target.Where(t => t.Type == APPLICATION_TYPE_MOBILE && String.Compare(t.Controller, jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE.Controller, StringComparison.InvariantCultureIgnoreCase) == 0 && String.Compare(t.Application, jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE.Application, StringComparison.InvariantCultureIgnoreCase) == 0).FirstOrDefault(); if (jobTargetReferenceApp == null) { // No valid reference, fall back to comparing against template logger.Warn("Unable to find reference target {0}, will index default template", jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE); loggerConsole.Warn("Unable to find reference target {0}, will index default template", jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE); jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE.Controller = BLANK_APPLICATION_CONTROLLER; jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE.Application = BLANK_APPLICATION_MOBILE; jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE.Type = APPLICATION_TYPE_MOBILE; jobConfiguration.Target.Add(jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE); } } #endregion bool reportFolderCleaned = false; // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_MOBILE) { continue; } StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Preload list of detected entities // For later cross-reference List <ControllerApplication> controllerApplicationList = FileIOHelper.ReadListFromCSVFile <ControllerApplication>(FilePathMap.ControllerApplicationsIndexFilePath(jobTarget), new ControllerApplicationReportMap()); #endregion #region Application Summary MOBILEApplicationConfiguration applicationConfiguration = new MOBILEApplicationConfiguration(); loggerConsole.Info("Application Summary"); applicationConfiguration.Controller = jobTarget.Controller; applicationConfiguration.ControllerLink = String.Format(DEEPLINK_CONTROLLER, applicationConfiguration.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); applicationConfiguration.ApplicationName = jobTarget.Application; applicationConfiguration.ApplicationID = jobTarget.ApplicationID; applicationConfiguration.ApplicationLink = String.Format(DEEPLINK_APM_APPLICATION, applicationConfiguration.Controller, applicationConfiguration.ApplicationID, DEEPLINK_TIMERANGE_LAST_15_MINUTES); if (controllerApplicationList != null) { ControllerApplication controllerApplication = controllerApplicationList.Where(a => a.Type == APPLICATION_TYPE_MOBILE && a.ApplicationID == jobTarget.ApplicationID).FirstOrDefault(); if (controllerApplication != null) { applicationConfiguration.ApplicationDescription = controllerApplication.Description; } } // Application Key JObject appKeyObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.MOBILEApplicationKeyDataFilePath(jobTarget)); if (appKeyObject != null) { applicationConfiguration.ApplicationKey = getStringValueFromJToken(appKeyObject, "appKey"); } // Monitoring State string monitoringState = FileIOHelper.ReadFileFromPath(FilePathMap.MOBILEApplicationMonitoringStateDataFilePath(jobTarget)); if (monitoringState != String.Empty) { bool parsedBool = false; Boolean.TryParse(monitoringState, out parsedBool); applicationConfiguration.IsEnabled = parsedBool; } // Configuration Settings JObject configurationSettingsObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.MOBILEAgentPageSettingsRulesDataFilePath(jobTarget)); if (configurationSettingsObject != null) { if (isTokenPropertyNull(configurationSettingsObject, "thresholds") == false) { applicationConfiguration.SlowThresholdType = getStringValueFromJToken(configurationSettingsObject["thresholds"]["slowThreshold"], "type"); applicationConfiguration.SlowThreshold = getIntValueFromJToken(configurationSettingsObject["thresholds"]["slowThreshold"], "value"); applicationConfiguration.VerySlowThresholdType = getStringValueFromJToken(configurationSettingsObject["thresholds"]["verySlowThreshold"], "type"); applicationConfiguration.VerySlowThreshold = getIntValueFromJToken(configurationSettingsObject["thresholds"]["verySlowThreshold"], "value"); applicationConfiguration.StallThresholdType = getStringValueFromJToken(configurationSettingsObject["thresholds"]["stallThreshold"], "type"); applicationConfiguration.StallThreshold = getIntValueFromJToken(configurationSettingsObject["thresholds"]["stallThreshold"], "value"); } applicationConfiguration.Percentiles = getStringValueOfObjectFromJToken(configurationSettingsObject, "percentileMetrics", true); applicationConfiguration.SessionTimeout = getIntValueFromJToken(configurationSettingsObject["sessionsMonitor"], "sessionTimeoutMins"); applicationConfiguration.CrashThreshold = getIntValueFromJToken(configurationSettingsObject["crashAlerts"], "threshold"); applicationConfiguration.IsIPDisplayed = getBoolValueFromJToken(configurationSettingsObject, "ipAddressDisplayed"); applicationConfiguration.EnableScreenshot = getBoolValueFromJToken(configurationSettingsObject["agentConfigData"], "enableScreenshot"); applicationConfiguration.AutoScreenshot = getBoolValueFromJToken(configurationSettingsObject["agentConfigData"], "autoScreenshot"); applicationConfiguration.UseCellular = getBoolValueFromJToken(configurationSettingsObject["agentConfigData"], "screenshotUseCellular"); } #endregion #region Network Requests List <MOBILENetworkRequestRule> networkRequestRulesList = new List <MOBILENetworkRequestRule>(128); JObject networkRequestRulesObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.MOBILEAgentNetworkRequestsRulesDataFilePath(jobTarget)); if (networkRequestRulesObject != null) { if (isTokenPropertyNull(networkRequestRulesObject, "customNamingIncludeRules") == false) { JArray includeRulesArray = (JArray)networkRequestRulesObject["customNamingIncludeRules"]; foreach (JObject includeRuleObject in includeRulesArray) { MOBILENetworkRequestRule networkRequestRule = fillNetworkRequestRule(includeRuleObject, jobTarget); if (networkRequestRule != null) { networkRequestRule.DetectionType = "INCLUDE"; networkRequestRulesList.Add(networkRequestRule); } } } if (isTokenPropertyNull(networkRequestRulesObject, "customNamingExcludeRules") == false) { JArray excludeRulesArray = (JArray)networkRequestRulesObject["customNamingExcludeRules"]; foreach (JObject excludeRuleObject in excludeRulesArray) { MOBILENetworkRequestRule networkRequestRule = fillNetworkRequestRule(excludeRuleObject, jobTarget); if (networkRequestRule != null) { networkRequestRule.DetectionType = "EXCLUDE"; networkRequestRulesList.Add(networkRequestRule); } } } } // Sort them networkRequestRulesList = networkRequestRulesList.OrderBy(o => o.DetectionType).ThenBy(o => o.Priority).ToList(); FileIOHelper.WriteListToCSVFile(networkRequestRulesList, new MOBILENetworkRequestRuleReportMap(), FilePathMap.MOBILENetworkRequestRulesIndexFilePath(jobTarget)); loggerConsole.Info("Completed {0} Rules", networkRequestRulesList.Count); #endregion #region Application Settings if (networkRequestRulesList != null) { applicationConfiguration.NumNetworkRulesInclude = networkRequestRulesList.Count(r => r.DetectionType == "INCLUDE"); applicationConfiguration.NumNetworkRulesExclude = networkRequestRulesList.Count(r => r.DetectionType == "EXCLUDE"); } List <MOBILEApplicationConfiguration> applicationConfigurationsList = new List <MOBILEApplicationConfiguration>(1); applicationConfigurationsList.Add(applicationConfiguration); FileIOHelper.WriteListToCSVFile(applicationConfigurationsList, new MOBILEApplicationConfigurationReportMap(), FilePathMap.MOBILEApplicationConfigurationIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + applicationConfigurationsList.Count; #endregion #region Combine All for Report CSV // If it is the first one, clear out the combined folder if (reportFolderCleaned == false) { FileIOHelper.DeleteFolder(FilePathMap.MOBILEConfigurationReportFolderPath()); Thread.Sleep(1000); FileIOHelper.CreateFolder(FilePathMap.WEBConfigurationReportFolderPath()); reportFolderCleaned = true; } // Append all the individual report files into one if (File.Exists(FilePathMap.MOBILEApplicationConfigurationIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.MOBILEApplicationConfigurationIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.MOBILEApplicationConfigurationReportFilePath(), FilePathMap.MOBILEApplicationConfigurationIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.MOBILENetworkRequestRulesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.MOBILENetworkRequestRulesIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.MOBILENetworkRequestRulesReportFilePath(), FilePathMap.MOBILENetworkRequestRulesIndexFilePath(jobTarget)); } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } // Remove all templates from the list jobConfiguration.Target.RemoveAll(t => t.Controller == BLANK_APPLICATION_CONTROLLER); return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
private int extractSnapshots( JobConfiguration jobConfiguration, JobTarget jobTarget, ControllerApi controllerApi, List <JToken> entityList, List <AppDRESTTier> tiersNodeJSList, bool progressToConsole) { int j = 0; foreach (JToken snapshot in entityList) { // Only do first in chain if ((bool)snapshot["firstInChain"] == true) { logger.Info("Retrieving snapshot for Application {0}, Tier {1}, Business Transaction {2}, RequestGUID {3}", jobTarget.Application, snapshot["applicationComponentName"], snapshot["businessTransactionName"], snapshot["requestGUID"]); #region Target step variables DateTime snapshotTime = UnixTimeHelper.ConvertFromUnixTimestamp((long)snapshot["serverStartTime"]); string snapshotFolderPath = FilePathMap.SnapshotDataFolderPath( jobTarget, snapshot["applicationComponentName"].ToString(), (long)snapshot["applicationComponentId"], snapshot["businessTransactionName"].ToString(), (long)snapshot["businessTransactionId"], snapshotTime, snapshot["userExperience"].ToString(), snapshot["requestGUID"].ToString()); // Must strip out the milliseconds, because the segment list retrieval doesn't seem to like them in the datetimes DateTime snapshotTimeFrom = snapshotTime.AddMinutes(-30).AddMilliseconds(snapshotTime.Millisecond * -1); DateTime snapshotTimeTo = snapshotTime.AddMinutes(30).AddMilliseconds(snapshotTime.Millisecond * -1); long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(snapshotTimeFrom); long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(snapshotTimeTo); int differenceInMinutes = (int)(snapshotTimeTo - snapshotTimeFrom).TotalMinutes; #endregion #region Get Snapshot Flowmap // Get snapshot flow map // Commenting this out until the time I decide to build visual representation of it, until then it is not needed //string snapshotFlowmapDataFilePath = Path.Combine(snapshotFolderPath, EXTRACT_SNAPSHOT_FLOWMAP_FILE_NAME); //if (File.Exists(snapshotFlowmapDataFilePath) == false) //{ // string snapshotFlowmapJson = controllerApi.GetFlowmapSnapshot(jobTarget.ApplicationID, (int)snapshot["businessTransactionId"], snapshot["requestGUID"].ToString(), fromTimeUnix, toTimeUnix, differenceInMinutes); // if (snapshotFlowmapJson != String.Empty) FileIOHelper.SaveFileToPath(snapshotFlowmapJson, snapshotFlowmapDataFilePath); //} #endregion #region Get List of Segments // Get list of segments string snapshotSegmentsDataFilePath = FilePathMap.SnapshotSegmentsDataFilePath(snapshotFolderPath); if (File.Exists(snapshotSegmentsDataFilePath) == false) { string snapshotSegmentsJson = controllerApi.GetSnapshotSegments(snapshot["requestGUID"].ToString(), snapshotTimeFrom, snapshotTimeTo, differenceInMinutes); if (snapshotSegmentsJson != String.Empty) { FileIOHelper.SaveFileToPath(snapshotSegmentsJson, snapshotSegmentsDataFilePath); } } #endregion #region Get Details for Each Segment JArray snapshotSegmentsList = FileIOHelper.LoadJArrayFromFile(snapshotSegmentsDataFilePath); if (snapshotSegmentsList != null) { // Get details for segment foreach (JToken snapshotSegment in snapshotSegmentsList) { string snapshotSegmentDataFilePath = FilePathMap.SnapshotSegmentDataFilePath(snapshotFolderPath, snapshotSegment["id"].ToString()); if (File.Exists(snapshotSegmentDataFilePath) == false) { string snapshotSegmentJson = controllerApi.GetSnapshotSegmentDetails((long)snapshotSegment["id"], fromTimeUnix, toTimeUnix, differenceInMinutes); if (snapshotSegmentJson != String.Empty) { FileIOHelper.SaveFileToPath(snapshotSegmentJson, snapshotSegmentDataFilePath); } } } // Get errors for segment foreach (JToken snapshotSegment in snapshotSegmentsList) { string snapshotSegmentErrorFilePath = FilePathMap.SnapshotSegmentErrorDataFilePath(snapshotFolderPath, snapshotSegment["id"].ToString()); if (File.Exists(snapshotSegmentErrorFilePath) == false && (bool)snapshotSegment["errorOccurred"] == true) { string snapshotSegmentJson = controllerApi.GetSnapshotSegmentErrors((long)snapshotSegment["id"], fromTimeUnix, toTimeUnix, differenceInMinutes); if (snapshotSegmentJson != String.Empty) { // "[ ]" == empty data. Don't create the file if (snapshotSegmentJson.Length > 3) { FileIOHelper.SaveFileToPath(snapshotSegmentJson, snapshotSegmentErrorFilePath); } } } } // Get call graphs for segment foreach (JToken snapshotSegment in snapshotSegmentsList) { string snapshotSegmentCallGraphFilePath = FilePathMap.SnapshotSegmentCallGraphDataFilePath(snapshotFolderPath, snapshotSegment["id"].ToString()); if (File.Exists(snapshotSegmentCallGraphFilePath) == false && ((bool)snapshotSegment["fullCallgraph"] == true || (bool)snapshotSegment["delayedCallGraph"] == true)) { // If the tier is Node.JS, the call graphs come from Process Snapshot bool getProcessCallGraph = false; string processRequestGUID = String.Empty; if (tiersNodeJSList != null && tiersNodeJSList.Count > 0) { // Is this a Node.JS tier? if (tiersNodeJSList.Count(t => t.id == (long)snapshotSegment["applicationComponentId"]) > 0) { // Yes, it is // Is there a process snapshot? Check Transaction Properties for its value string snapshotSegmentDataFilePath = FilePathMap.SnapshotSegmentDataFilePath(snapshotFolderPath, snapshotSegment["id"].ToString()); JObject snapshotSegmentDetail = FileIOHelper.LoadJObjectFromFile(snapshotSegmentDataFilePath); if (snapshotSegmentDetail != null) { if (snapshotSegmentDetail["transactionProperties"].HasValues == true) { foreach (JToken transactionPropertyToken in snapshotSegmentDetail["transactionProperties"]) { if (transactionPropertyToken["name"].ToString() == "Process Snapshot GUIDs") { getProcessCallGraph = true; processRequestGUID = transactionPropertyToken["value"].ToString(); break; } } } } } } // Ok, now either get call graph the usual way or process snapshot call graph if (getProcessCallGraph == true && processRequestGUID.Length > 0) { string snapshotSegmentJson = controllerApi.GetProcessSnapshotCallGraph(processRequestGUID, fromTimeUnix, toTimeUnix, differenceInMinutes); if (snapshotSegmentJson != String.Empty) { FileIOHelper.SaveFileToPath(snapshotSegmentJson, snapshotSegmentCallGraphFilePath); } } else { string snapshotSegmentJson = controllerApi.GetSnapshotSegmentCallGraph((long)snapshotSegment["id"], fromTimeUnix, toTimeUnix, differenceInMinutes); if (snapshotSegmentJson != String.Empty) { FileIOHelper.SaveFileToPath(snapshotSegmentJson, snapshotSegmentCallGraphFilePath); } } } } } #endregion } if (progressToConsole == true) { j++; if (j % 10 == 0) { Console.Write("[{0}].", j); } } } return(entityList.Count); }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(jobConfiguration) == false) { return(true); } // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Target state check if (jobTarget.Status != JobTargetStatus.ConfigurationValid) { loggerConsole.Trace("Target in invalid state {0}, skipping", jobTarget.Status); continue; } #endregion #region Target step variables // Set up controller access ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)); // Login into private API controllerApi.PrivateApiLogin(); #endregion #region Get list of Snapshots in time ranges loggerConsole.Info("Extract List of Snapshots ({0} time ranges)", jobConfiguration.Input.HourlyTimeRanges.Count); // Get list of snapshots in each time range int totalSnapshotsFound = 0; foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges) { logger.Info("Extract List of Snapshots from {0:o} to {1:o}", jobTimeRange.From, jobTimeRange.To); loggerConsole.Info("Extract List of Snapshots from {0:G} to {1:G}", jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime()); string snapshotsDataFilePath = FilePathMap.SnapshotsDataFilePath(jobTarget, jobTimeRange); int differenceInMinutes = (int)(jobTimeRange.To - jobTimeRange.From).TotalMinutes; if (File.Exists(snapshotsDataFilePath) == false) { JArray listOfSnapshots = new JArray(); // Extract snapshot list long serverCursorId = 0; string serverCursorIdType = String.Empty; do { string snapshotsJSON = String.Empty; if (serverCursorId == 0) { // Extract first page of snapshots snapshotsJSON = controllerApi.GetListOfSnapshotsFirstPage(jobTarget.ApplicationID, jobTimeRange.From, jobTimeRange.To, differenceInMinutes, SNAPSHOTS_QUERY_PAGE_SIZE); } else { // If there are more snapshots on the server, the server cursor would be non-0 switch (serverCursorIdType) { case "scrollId": // Sometimes - >4.3.3? the value of scroll is in scrollId, not rsdScrollId // "serverCursor" : { // "scrollId" : 1509543646696 // } snapshotsJSON = controllerApi.GetListOfSnapshotsNextPage_Type_scrollId(jobTarget.ApplicationID, jobTimeRange.From, jobTimeRange.To, differenceInMinutes, SNAPSHOTS_QUERY_PAGE_SIZE, serverCursorId); break; case "rsdScrollId": // "serverCursor" : { // "rsdScrollId" : 1509543646696 // } snapshotsJSON = controllerApi.GetListOfSnapshotsNextPage_Type_rsdScrollId(jobTarget.ApplicationID, jobTimeRange.From, jobTimeRange.To, differenceInMinutes, SNAPSHOTS_QUERY_PAGE_SIZE, serverCursorId); break; case "fetchMoreDataHandle": // Seen this on 4.2.3.0 Controller. Maybe that's how it used to be? // "fetchMoreDataHandle":1509626881987 // Can't seem to make it load more than 600 items snapshotsJSON = controllerApi.GetListOfSnapshotsNextPage_Type_handle(jobTarget.ApplicationID, jobTimeRange.From, jobTimeRange.To, differenceInMinutes, SNAPSHOTS_QUERY_PAGE_SIZE, serverCursorId); break; default: logger.Warn("Unknown type of serverCursorIdType={0}, not going to retrieve any snapshots", serverCursorIdType); break; } } // Assume we have no more pages serverCursorId = 0; // Process retrieved snapshots and check if we actually have more pages if (snapshotsJSON != String.Empty) { Console.Write("."); // Load snapshots into array JObject snapshotsParsed = JObject.Parse(snapshotsJSON); JArray snapshots = (JArray)snapshotsParsed["requestSegmentDataListItems"]; foreach (JObject snapshot in snapshots) { listOfSnapshots.Add(snapshot); } // Check whether we have more snapshots and if yes, get continuation type and cursor ID JToken fetchMoreDataHandleObj = snapshotsParsed["fetchMoreDataHandle"]; JToken serverCursorObj = snapshotsParsed["serverCursor"]; if (serverCursorObj != null) { JToken scrollIdObj = serverCursorObj["scrollId"]; JToken rsdScrollIdObj = serverCursorObj["rsdScrollId"]; if (scrollIdObj != null) { serverCursorIdType = "scrollId"; // Parse the cursor ID if (Int64.TryParse(scrollIdObj.ToString(), out serverCursorId) == false) { // Nope, not going to go forward serverCursorId = 0; } } else if (rsdScrollIdObj != null) { serverCursorIdType = "rsdScrollId"; // Parse the cursor ID if (Int64.TryParse(rsdScrollIdObj.ToString(), out serverCursorId) == false) { // Nope, not going to go forward serverCursorId = 0; } } } else if (fetchMoreDataHandleObj != null) { serverCursorIdType = "fetchMoreDataHandle"; // Parse the cursor ID if (Int64.TryParse(fetchMoreDataHandleObj.ToString(), out serverCursorId) == false) { // Nope, not going to go forward serverCursorId = 0; } } else { logger.Warn("Snapshot list retrival call unexpectedly did not have any evidence of continuation CursorId"); } logger.Info("Retrieved snapshots from Controller {0}, Application {1}, From {2:o}, To {3:o}', number of snapshots {4}, continuation type {5}, continuation CursorId {6}", jobTarget.Controller, jobTarget.Application, jobTimeRange.From, jobTimeRange.To, snapshots.Count, serverCursorIdType, serverCursorId); // Move to next loop Console.Write("+{0}", listOfSnapshots.Count); } }while (serverCursorId > 0); Console.WriteLine(); FileIOHelper.WriteJArrayToFile(listOfSnapshots, snapshotsDataFilePath); totalSnapshotsFound = totalSnapshotsFound + listOfSnapshots.Count; logger.Info("{0} snapshots from {1:o} to {2:o}", listOfSnapshots.Count, jobTimeRange.From, jobTimeRange.To); loggerConsole.Info("{0} snapshots from {1:G} to {2:G}", listOfSnapshots.Count, jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime()); } } logger.Info("{0} snapshots in all time ranges", totalSnapshotsFound); loggerConsole.Info("{0} snapshots in all time ranges", totalSnapshotsFound); #endregion #region Get individual Snapshots // Extract individual snapshots loggerConsole.Info("Extract Individual Snapshots"); List <AppDRESTTier> tiersList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTTier>(FilePathMap.TiersDataFilePath(jobTarget)); List <AppDRESTBusinessTransaction> businessTransactionsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBusinessTransaction>(FilePathMap.BusinessTransactionsDataFilePath(jobTarget)); // Identify Node.JS tiers that will extact call graph using a different call List <AppDRESTTier> tiersNodeJSList = null; if (tiersList != null) { tiersNodeJSList = tiersList.Where(t => t.agentType == "NODEJS_APP_AGENT").ToList(); } // Process each hour at a time foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges) { string snapshotsDataFilePath = FilePathMap.SnapshotsDataFilePath(jobTarget, jobTimeRange); JArray listOfSnapshotsInHour = FileIOHelper.LoadJArrayFromFile(snapshotsDataFilePath); if (listOfSnapshotsInHour != null && listOfSnapshotsInHour.Count > 0) { logger.Info("Filter Snapshots {0:o} to {1:o} ({2} snapshots)", jobTimeRange.From, jobTimeRange.To, listOfSnapshotsInHour.Count); loggerConsole.Info("Filter Snapshots {0:G} to {1:G} ({2} snapshots)", jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime(), listOfSnapshotsInHour.Count); // Filter the list of snapshots based on SnapshotSelectionCriteria List <JToken> listOfSnapshotsInHourFiltered = new List <JToken>(listOfSnapshotsInHour.Count); foreach (JToken snapshotToken in listOfSnapshotsInHour) { logger.Trace("Considering filtering snapshot requestGUID={0}, firstInChain={1}, userExperience={2}, fullCallgraph={3}, delayedCallGraph={4}, applicationComponentName={5}, businessTransactionName={6}", snapshotToken["requestGUID"], snapshotToken["firstInChain"], snapshotToken["userExperience"], snapshotToken["fullCallgraph"], snapshotToken["delayedCallGraph"], snapshotToken["applicationComponentName"], snapshotToken["businessTransactionName"]); // Only grab first in chain snapshots if ((bool)snapshotToken["firstInChain"] == false) { continue; } // Filter user experience switch (snapshotToken["userExperience"].ToString()) { case "NORMAL": if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Normal != true) { continue; } break; case "SLOW": if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Slow != true) { continue; } break; case "VERY_SLOW": if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.VerySlow != true) { continue; } break; case "STALL": if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Stall != true) { continue; } break; case "ERROR": if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Error != true) { continue; } break; default: // Not sure what kind of beast it is continue; } // Filter call graph if ((bool)snapshotToken["fullCallgraph"] == true) { if (jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType.Full != true) { continue; } } else if ((bool)snapshotToken["delayedCallGraph"] == true) { if (jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType.Partial != true) { continue; } } else { if (jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType.None != true) { continue; } } // Filter Tier type if (jobConfiguration.Input.SnapshotSelectionCriteria.TierType.All != true) { if (tiersList != null) { AppDRESTTier tier = tiersList.Where(t => t.id == (long)snapshotToken["applicationComponentId"]).FirstOrDefault(); if (tier != null) { PropertyInfo pi = jobConfiguration.Input.SnapshotSelectionCriteria.TierType.GetType().GetProperty(tier.agentType); if (pi != null) { if ((bool)pi.GetValue(jobConfiguration.Input.SnapshotSelectionCriteria.TierType) == false) { continue; } } } } } // Filter BT type if (jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactionType.All != true) { if (businessTransactionsList != null) { AppDRESTBusinessTransaction businessTransaction = businessTransactionsList.Where(b => b.id == (long)snapshotToken["businessTransactionId"] && b.tierId == (long)snapshotToken["applicationComponentId"]).FirstOrDefault(); if (businessTransaction != null) { PropertyInfo pi = jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactionType.GetType().GetProperty(businessTransaction.entryPointType); if (pi != null) { if ((bool)pi.GetValue(jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactionType) == false) { continue; } } } } } // Filter Tier name bool tierNameMatch = false; if (jobConfiguration.Input.SnapshotSelectionCriteria.Tiers.Length == 0) { tierNameMatch = true; } foreach (string matchCriteria in jobConfiguration.Input.SnapshotSelectionCriteria.Tiers) { if (matchCriteria.Length > 0) { // Try straight up string compare first if (String.Compare(snapshotToken["applicationComponentName"].ToString(), matchCriteria, true) == 0) { tierNameMatch = true; break; } // Try regex compare second Regex regexQuery = new Regex(matchCriteria, RegexOptions.IgnoreCase); Match regexMatch = regexQuery.Match(snapshotToken["applicationComponentName"].ToString()); if (regexMatch.Success == true && regexMatch.Index == 0) { tierNameMatch = true; break; } } } if (tierNameMatch == false) { continue; } // Filter BT name bool businessTransactionNameMatch = false; if (jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactions.Length == 0) { businessTransactionNameMatch = true; } foreach (string matchCriteria in jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactions) { if (matchCriteria.Length > 0) { // Try straight up string compare first if (String.Compare(snapshotToken["businessTransactionName"].ToString(), matchCriteria, true) == 0) { businessTransactionNameMatch = true; break; } // Try regex compare second Regex regexQuery = new Regex(matchCriteria, RegexOptions.IgnoreCase); Match regexMatch = regexQuery.Match(snapshotToken["businessTransactionName"].ToString()); if (regexMatch.Success == true && regexMatch.Index == 0) { businessTransactionNameMatch = true; break; } } } if (businessTransactionNameMatch == false) { continue; } // If we got here, then the snapshot passed the filter logger.Trace("Keeping snapshot requestGUID={0}, firstInChain={1}, userExperience={2}, fullCallgraph={3}, delayedCallGraph={4}, applicationComponentName={5}, businessTransactionName={6}", snapshotToken["requestGUID"], snapshotToken["firstInChain"], snapshotToken["userExperience"], snapshotToken["fullCallgraph"], snapshotToken["delayedCallGraph"], snapshotToken["applicationComponentName"], snapshotToken["businessTransactionName"]); listOfSnapshotsInHourFiltered.Add(snapshotToken); } logger.Info("Total Snapshots {0:o} to {1:o} is {2}, after filtered {3}", jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime(), listOfSnapshotsInHour.Count, listOfSnapshotsInHourFiltered.Count); // Now extract things logger.Info("Extract Snapshots {0:o} to {1:o} ({2} snapshots)", jobTimeRange.From, jobTimeRange.To, listOfSnapshotsInHourFiltered.Count); loggerConsole.Info("Extract Snapshots {0:G} to {1:G} ({2} snapshots)", jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime(), listOfSnapshotsInHourFiltered.Count); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + listOfSnapshotsInHourFiltered.Count; int numSnapshots = 0; if (programOptions.ProcessSequentially == false) { var listOfSnapshotsInHourChunks = listOfSnapshotsInHourFiltered.BreakListIntoChunks(SNAPSHOTS_EXTRACT_NUMBER_OF_ENTITIES_TO_PROCESS_PER_THREAD); Parallel.ForEach <List <JToken>, int>( listOfSnapshotsInHourChunks, new ParallelOptions { MaxDegreeOfParallelism = SNAPSHOTS_EXTRACT_NUMBER_OF_THREADS }, () => 0, (listOfSnapshotsInHourChunk, loop, subtotal) => { // Set up controller access ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)); // Login into private API controllerApiParallel.PrivateApiLogin(); subtotal += extractSnapshots(jobConfiguration, jobTarget, controllerApiParallel, listOfSnapshotsInHourChunk, tiersNodeJSList, false); return(subtotal); }, (finalResult) => { Interlocked.Add(ref numSnapshots, finalResult); Console.Write("[{0}].", numSnapshots); } ); } else { numSnapshots = extractSnapshots(jobConfiguration, jobTarget, controllerApi, listOfSnapshotsInHourFiltered, tiersNodeJSList, true); } loggerConsole.Info("{0} snapshots", numSnapshots); } } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(jobConfiguration) == false) { return(true); } // Process each Controller once int i = 0; var controllers = jobConfiguration.Target.GroupBy(t => t.Controller); foreach (var controllerGroup in controllers) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = controllerGroup.ToList()[0]; StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; stepTimingTarget.NumEntities = 1; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); // Set up controller access using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { #region Controller loggerConsole.Info("Controller Version"); string controllerVersionXML = controllerApi.GetControllerVersion(); if (controllerVersionXML != String.Empty) { FileIOHelper.SaveFileToPath(controllerVersionXML, FilePathMap.ControllerVersionDataFilePath(jobTarget)); } #endregion #region APM Applications loggerConsole.Info("APM Applications"); string applicationsJSON = controllerApi.GetAPMApplications(); if (applicationsJSON != String.Empty) { FileIOHelper.SaveFileToPath(applicationsJSON, FilePathMap.APMApplicationsDataFilePath(jobTarget)); } #endregion #region Applications loggerConsole.Info("All Applications"); controllerApi.PrivateApiLogin(); string applicationsAllJSON = controllerApi.GetAllApplicationsAllTypes(); if (applicationsAllJSON != String.Empty) { FileIOHelper.SaveFileToPath(applicationsAllJSON, FilePathMap.AllApplicationsDataFilePath(jobTarget)); } #endregion #region Mobile Applications loggerConsole.Info("Mobile Applications"); string applicationsMobileJSON = controllerApi.GetMOBILEApplications(); if (applicationsMobileJSON != String.Empty) { FileIOHelper.SaveFileToPath(applicationsMobileJSON, FilePathMap.MOBILEApplicationsDataFilePath(jobTarget)); } #endregion } } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } i++; } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(programOptions, jobConfiguration) == false) { return(true); } if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_APM) == 0) { logger.Warn("No {0} targets to process", APPLICATION_TYPE_APM); loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_APM); return(true); } // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_APM) { continue; } StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Target step variables int numEntitiesTotal = 0; #endregion #region Prepare time range long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From); long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To); long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000); #endregion ParallelOptions parallelOptions = new ParallelOptions(); if (programOptions.ProcessSequentially == true) { parallelOptions.MaxDegreeOfParallelism = 1; } Parallel.Invoke(parallelOptions, () => { #region Application loggerConsole.Info("Extract Flowmap for Application"); using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { controllerApi.PrivateApiLogin(); if (File.Exists(FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange)) == false) { string flowmapJson = controllerApi.GetFlowmapApplication(jobTarget.ApplicationID, fromTimeUnix, toTimeUnix, differenceInMinutes); if (flowmapJson != String.Empty) { FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange)); } } loggerConsole.Info("Completed Application"); } // Removing as this doesn't seem to be a helpful report //JobTimeRange jobTimeRangeLast = jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1]; //int differenceInMinutesForLastTimeRange = (int)((jobTimeRangeLast.To - jobTimeRangeLast.From).TotalMinutes); //loggerConsole.Info("Extract Flowmap for Application in each minute in in last timerange ({0} minutes)", differenceInMinutesForLastTimeRange); //int j = 0; //Parallel.For(0, // differenceInMinutesForLastTimeRange, // parallelOptions, // () => 0, // (minute, loop, subtotal) => // { // using (ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) // { // controllerApiLocal.PrivateApiLogin(); // JobTimeRange thisMinuteJobTimeRange = new JobTimeRange(); // thisMinuteJobTimeRange.From = jobTimeRangeLast.From.AddMinutes(minute); // thisMinuteJobTimeRange.To = jobTimeRangeLast.From.AddMinutes(minute + 1); // long fromTimeUnixLocal = UnixTimeHelper.ConvertToUnixTimestamp(thisMinuteJobTimeRange.From); // long toTimeUnixLocal = UnixTimeHelper.ConvertToUnixTimestamp(thisMinuteJobTimeRange.To); // long differenceInMinutesLocal = 1; // if (File.Exists(FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, thisMinuteJobTimeRange)) == false) // { // string flowmapJson = controllerApiLocal.GetFlowmapApplication(jobTarget.ApplicationID, fromTimeUnixLocal, toTimeUnixLocal, differenceInMinutesLocal); // if (flowmapJson != String.Empty) FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, thisMinuteJobTimeRange)); // } // return 1; // } // }, // (finalResult) => // { // Interlocked.Add(ref j, finalResult); // if (j % 10 == 0) // { // Console.Write("[{0}].", j); // } // } //); //loggerConsole.Info("Completed Application in each minute {0} minutes", differenceInMinutesForLastTimeRange); Interlocked.Add(ref numEntitiesTotal, 1); #endregion }, () => { #region Tiers List <AppDRESTTier> tiersList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTTier>(FilePathMap.APMTiersDataFilePath(jobTarget)); if (tiersList != null) { loggerConsole.Info("Extract Flowmaps for Tiers ({0} entities)", tiersList.Count); int j = 0; Parallel.ForEach( tiersList, parallelOptions, () => 0, (tier, loop, subtotal) => { using (ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { controllerApiLocal.PrivateApiLogin(); if (File.Exists(FilePathMap.TierFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, tier)) == false) { string flowmapJson = controllerApiLocal.GetFlowmapTier(tier.id, fromTimeUnix, toTimeUnix, differenceInMinutes); if (flowmapJson != String.Empty) { FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.TierFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, tier)); } } return(1); } }, (finalResult) => { Interlocked.Add(ref j, finalResult); if (j % 10 == 0) { Console.Write("[{0}].", j); } } ); loggerConsole.Info("Completed {0} Tiers", tiersList.Count); Interlocked.Add(ref numEntitiesTotal, tiersList.Count); } #endregion }, () => { #region Nodes List <AppDRESTNode> nodesList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTNode>(FilePathMap.APMNodesDataFilePath(jobTarget)); if (nodesList != null) { loggerConsole.Info("Extract Flowmaps for Nodes ({0} entities)", nodesList.Count); int j = 0; Parallel.ForEach( nodesList, parallelOptions, () => 0, (node, loop, subtotal) => { using (ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { controllerApiLocal.PrivateApiLogin(); if (File.Exists(FilePathMap.NodeFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, node)) == false) { string flowmapJson = controllerApiLocal.GetFlowmapNode(node.id, fromTimeUnix, toTimeUnix, differenceInMinutes); if (flowmapJson != String.Empty) { FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.NodeFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, node)); } } return(1); } }, (finalResult) => { Interlocked.Add(ref j, finalResult); if (j % 10 == 0) { Console.Write("[{0}].", j); } } ); loggerConsole.Info("Completed {0} Nodes", nodesList.Count); Interlocked.Add(ref numEntitiesTotal, nodesList.Count); } #endregion }, () => { #region Backends List <AppDRESTBackend> backendsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBackend>(FilePathMap.APMBackendsDataFilePath(jobTarget)); if (backendsList != null) { loggerConsole.Info("Extract Flowmaps for Backends ({0} entities)", backendsList.Count); int j = 0; Parallel.ForEach( backendsList, parallelOptions, () => 0, (backend, loop, subtotal) => { ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)); controllerApiLocal.PrivateApiLogin(); if (File.Exists(FilePathMap.BackendFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, backend)) == false) { string flowmapJson = controllerApiLocal.GetFlowmapBackend(backend.id, fromTimeUnix, toTimeUnix, differenceInMinutes); if (flowmapJson != String.Empty) { FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.BackendFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, backend)); } } return(1); }, (finalResult) => { Interlocked.Add(ref j, finalResult); if (j % 10 == 0) { Console.Write("[{0}].", j); } } ); loggerConsole.Info("Completed {0} Backends", backendsList.Count); Interlocked.Add(ref numEntitiesTotal, backendsList.Count); } #endregion }, () => { #region Business Transactions List <AppDRESTBusinessTransaction> businessTransactionsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBusinessTransaction>(FilePathMap.APMBusinessTransactionsDataFilePath(jobTarget)); if (businessTransactionsList != null) { loggerConsole.Info("Extract Flowmaps for Business Transactions ({0} entities)", businessTransactionsList.Count); int j = 0; Parallel.ForEach( businessTransactionsList, parallelOptions, () => 0, (businessTransaction, loop, subtotal) => { using (ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { controllerApiLocal.PrivateApiLogin(); if (File.Exists(FilePathMap.BusinessTransactionFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, businessTransaction)) == false) { string flowmapJson = controllerApiLocal.GetFlowmapBusinessTransaction(jobTarget.ApplicationID, businessTransaction.id, fromTimeUnix, toTimeUnix, differenceInMinutes); if (flowmapJson != String.Empty) { FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.BusinessTransactionFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, businessTransaction)); } } return(1); } }, (finalResult) => { Interlocked.Add(ref j, finalResult); if (j % 10 == 0) { Console.Write("[{0}].", j); } } ); loggerConsole.Info("Completed {0} Business Transactions", businessTransactionsList.Count); Interlocked.Add(ref numEntitiesTotal, businessTransactionsList.Count); } #endregion } ); stepTimingTarget.NumEntities = numEntitiesTotal; } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(programOptions, jobConfiguration) == false) { return(true); } if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_SIM) == 0) { logger.Warn("No {0} targets to process", APPLICATION_TYPE_SIM); loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_SIM); return(true); } List <MetricExtractMapping> entityMetricExtractMappingList = getMetricsExtractMappingList(jobConfiguration); // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_SIM) { continue; } StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Target step variables stepTimingTarget.NumEntities = entityMetricExtractMappingList.Count; #endregion loggerConsole.Info("Extract Metrics for All Entities ({0} time ranges)", jobConfiguration.Input.HourlyTimeRanges.Count); ParallelOptions parallelOptions = new ParallelOptions(); if (programOptions.ProcessSequentially == true) { parallelOptions.MaxDegreeOfParallelism = 1; } Parallel.Invoke(parallelOptions, () => { #region Machine getMetricsForEntities(jobTarget, jobConfiguration, entityMetricExtractMappingList, SIMMachine.ENTITY_FOLDER, SIMMachine.ENTITY_TYPE); #endregion }, () => { #region Network getMetricsForEntities(jobTarget, jobConfiguration, entityMetricExtractMappingList, SIMMachineNetwork.ENTITY_FOLDER, SIMMachineNetwork.ENTITY_TYPE); #endregion } ); } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }