public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(programOptions, jobConfiguration) == false) { return(true); } bool reportFolderCleaned = false; // Process each Controller once int i = 0; var controllers = jobConfiguration.Target.GroupBy(t => t.Controller); foreach (var controllerGroup in controllers) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = controllerGroup.ToList()[0]; StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Target step variables List <HealthCheckRuleResult> healthCheckRuleResults = new List <HealthCheckRuleResult>(); #endregion #region Preload all the reports that will be filtered by the subsequent entities loggerConsole.Info("Entity Details Data Preloading"); // This file will always be there List <HealthCheckSettingMapping> healthCheckSettingsList = FileIOHelper.ReadListFromCSVFile <HealthCheckSettingMapping>(FilePathMap.HealthCheckSettingMappingFilePath(), new HealthCheckSettingMappingMap()); if (healthCheckSettingsList == null || healthCheckSettingsList.Count == 0) { loggerConsole.Warn("Health check settings file did not load. Exiting the health checks"); return(false); } Dictionary <string, HealthCheckSettingMapping> healthCheckSettingsDictionary = healthCheckSettingsList.ToDictionary(h => h.Name, h => h); List <ControllerSummary> controllerSummariesList = FileIOHelper.ReadListFromCSVFile <ControllerSummary>(FilePathMap.ControllerSummaryIndexFilePath(jobTarget), new ControllerSummaryReportMap()); List <ControllerSetting> controllerSettingsList = FileIOHelper.ReadListFromCSVFile <ControllerSetting>(FilePathMap.ControllerSettingsIndexFilePath(jobTarget), new ControllerSettingReportMap()); #endregion #region Controller Version and Properties healthCheckRuleResults.Add( evaluate_Controller_Version( new HealthCheckRuleDescription("Platform", "PLAT-001-PLATFORM-VERSION", "Controller Version"), jobTarget, healthCheckSettingsDictionary, controllerSummariesList)); healthCheckRuleResults.Add( evaluate_Controller_SaaS_OnPrem( new HealthCheckRuleDescription("Platform", "PLAT-002-PLATFORM-SAAS", "SaaS or OnPrem"), jobTarget, healthCheckSettingsDictionary)); healthCheckRuleResults.Add( evaluate_Controller_Setting_Performance_Profile( new HealthCheckRuleDescription("Platform", "PLAT-003-PLATFORM-PERFORMANCE-PROFILE", "Performance Profile"), jobTarget, healthCheckSettingsDictionary, controllerSettingsList)); healthCheckRuleResults.AddRange( evaluate_Controller_Setting_Buffer_Sizes( new HealthCheckRuleDescription("Platform", "PLAT-004-PLATFORM-BUFFER-SIZES", "Buffer Size"), jobTarget, healthCheckSettingsDictionary, controllerSettingsList)); healthCheckRuleResults.AddRange( evaluate_Controller_Setting_ADD_Limits( new HealthCheckRuleDescription("Platform", "PLAT-005-PLATFORM-ADD-LIMITS", "ADD Limit"), jobTarget, healthCheckSettingsDictionary, controllerSettingsList)); // TODO Add things like ADD limits, etc #endregion // Remove any health rule results that weren't very good healthCheckRuleResults.RemoveAll(h => h == null); // Sort them healthCheckRuleResults = healthCheckRuleResults.OrderBy(h => h.EntityType).ThenBy(h => h.EntityName).ThenBy(h => h.Category).ThenBy(h => h.Name).ToList(); // Set version to each of the health check rule results string versionOfDEXTER = Assembly.GetEntryAssembly().GetName().Version.ToString(); foreach (HealthCheckRuleResult healthCheckRuleResult in healthCheckRuleResults) { healthCheckRuleResult.Version = versionOfDEXTER; } FileIOHelper.WriteListToCSVFile(healthCheckRuleResults, new HealthCheckRuleResultReportMap(), FilePathMap.ControllerHealthCheckRuleResultsIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = healthCheckRuleResults.Count; #region Combine All for Report CSV // If it is the first one, clear out the combined folder if (reportFolderCleaned == false) { FileIOHelper.DeleteFolder(FilePathMap.ControllerHealthCheckReportFolderPath()); Thread.Sleep(1000); FileIOHelper.CreateFolder(FilePathMap.ControllerHealthCheckReportFolderPath()); reportFolderCleaned = true; } // Append all the individual report files into one if (File.Exists(FilePathMap.ControllerHealthCheckRuleResultsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.ControllerHealthCheckRuleResultsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.ControllerHealthCheckRuleResultsReportFilePath(), FilePathMap.ControllerHealthCheckRuleResultsIndexFilePath(jobTarget)); } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(programOptions, jobConfiguration) == false) { return(true); } List <JobTarget> listOfTargetsAlreadyProcessed = new List <JobTarget>(jobConfiguration.Target.Count); bool reportFolderCleaned = false; bool haveProcessedAtLeastOneDBCollector = false; // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); if (listOfTargetsAlreadyProcessed.Count(j => (j.Controller == jobTarget.Controller) && (j.ApplicationID == jobTarget.ApplicationID)) > 0) { // Already saw this target, like an APM and WEB pairs together continue; } // For databases, we only process this once for the first collector we've seen if (jobTarget.Type == APPLICATION_TYPE_DB) { if (haveProcessedAtLeastOneDBCollector == false) { haveProcessedAtLeastOneDBCollector = true; } else { continue; } } listOfTargetsAlreadyProcessed.Add(jobTarget); #region Prepare time variables long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From); long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To); long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000); string DEEPLINK_THIS_TIMERANGE = String.Format(DEEPLINK_TIMERANGE_BETWEEN_TIMES, toTimeUnix, fromTimeUnix, differenceInMinutes); #endregion #region Health Rule violations List <HealthRuleViolationEvent> healthRuleViolationList = new List <HealthRuleViolationEvent>(); loggerConsole.Info("Index Health Rule Violations"); JArray healthRuleViolationsArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.ApplicationHealthRuleViolationsDataFilePath(jobTarget)); if (healthRuleViolationsArray != null) { foreach (JObject interestingEventObject in healthRuleViolationsArray) { HealthRuleViolationEvent healthRuleViolationEvent = new HealthRuleViolationEvent(); healthRuleViolationEvent.Controller = jobTarget.Controller; healthRuleViolationEvent.ApplicationName = jobTarget.Application; healthRuleViolationEvent.ApplicationID = jobTarget.ApplicationID; healthRuleViolationEvent.EventID = getLongValueFromJToken(interestingEventObject, "id"); healthRuleViolationEvent.FromUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(interestingEventObject, "startTimeInMillis")); healthRuleViolationEvent.From = healthRuleViolationEvent.FromUtc.ToLocalTime(); if (getLongValueFromJToken(interestingEventObject, "endTimeInMillis") > 0) { healthRuleViolationEvent.ToUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(interestingEventObject, "endTimeInMillis")); healthRuleViolationEvent.To = healthRuleViolationEvent.FromUtc.ToLocalTime(); } healthRuleViolationEvent.Status = getStringValueFromJToken(interestingEventObject, "incidentStatus"); healthRuleViolationEvent.Severity = getStringValueFromJToken(interestingEventObject, "severity"); healthRuleViolationEvent.EventLink = String.Format(DEEPLINK_INCIDENT, healthRuleViolationEvent.Controller, healthRuleViolationEvent.ApplicationID, healthRuleViolationEvent.EventID, getStringValueFromJToken(interestingEventObject, "startTimeInMillis"), DEEPLINK_THIS_TIMERANGE);; healthRuleViolationEvent.Description = getStringValueFromJToken(interestingEventObject, "description"); if (isTokenPropertyNull(interestingEventObject, "triggeredEntityDefinition") == false) { healthRuleViolationEvent.HealthRuleID = getLongValueFromJToken(interestingEventObject["triggeredEntityDefinition"], "entityId"); healthRuleViolationEvent.HealthRuleName = getStringValueFromJToken(interestingEventObject["triggeredEntityDefinition"], "name"); // TODO the health rule can't be hotlinked to until platform rewrites the screen that opens from Flash healthRuleViolationEvent.HealthRuleLink = String.Format(DEEPLINK_HEALTH_RULE, healthRuleViolationEvent.Controller, healthRuleViolationEvent.ApplicationID, healthRuleViolationEvent.HealthRuleID, DEEPLINK_THIS_TIMERANGE); } if (isTokenPropertyNull(interestingEventObject, "affectedEntityDefinition") == false) { healthRuleViolationEvent.EntityID = getIntValueFromJToken(interestingEventObject["affectedEntityDefinition"], "entityId"); healthRuleViolationEvent.EntityName = getStringValueFromJToken(interestingEventObject["affectedEntityDefinition"], "name"); string entityType = getStringValueFromJToken(interestingEventObject["affectedEntityDefinition"], "entityType"); if (entityTypeStringMapping.ContainsKey(entityType) == true) { healthRuleViolationEvent.EntityType = entityTypeStringMapping[entityType]; } else { healthRuleViolationEvent.EntityType = entityType; } // Come up with links switch (entityType) { case ENTITY_TYPE_FLOWMAP_APPLICATION: healthRuleViolationEvent.EntityLink = String.Format(DEEPLINK_APM_APPLICATION, healthRuleViolationEvent.Controller, healthRuleViolationEvent.ApplicationID, DEEPLINK_THIS_TIMERANGE); break; case ENTITY_TYPE_FLOWMAP_APPLICATION_MOBILE: healthRuleViolationEvent.EntityLink = String.Format(DEEPLINK_APPLICATION_MOBILE, healthRuleViolationEvent.Controller, healthRuleViolationEvent.ApplicationID, healthRuleViolationEvent.EntityID, DEEPLINK_THIS_TIMERANGE); break; case ENTITY_TYPE_FLOWMAP_TIER: healthRuleViolationEvent.EntityLink = String.Format(DEEPLINK_TIER, healthRuleViolationEvent.Controller, healthRuleViolationEvent.ApplicationID, healthRuleViolationEvent.EntityID, DEEPLINK_THIS_TIMERANGE); break; case ENTITY_TYPE_FLOWMAP_NODE: healthRuleViolationEvent.EntityLink = String.Format(DEEPLINK_NODE, healthRuleViolationEvent.Controller, healthRuleViolationEvent.ApplicationID, healthRuleViolationEvent.EntityID, DEEPLINK_THIS_TIMERANGE); break; case ENTITY_TYPE_FLOWMAP_BUSINESS_TRANSACTION: healthRuleViolationEvent.EntityLink = String.Format(DEEPLINK_BUSINESS_TRANSACTION, healthRuleViolationEvent.Controller, healthRuleViolationEvent.ApplicationID, healthRuleViolationEvent.EntityID, DEEPLINK_THIS_TIMERANGE); break; case ENTITY_TYPE_FLOWMAP_BACKEND: healthRuleViolationEvent.EntityLink = String.Format(DEEPLINK_BACKEND, healthRuleViolationEvent.Controller, healthRuleViolationEvent.ApplicationID, healthRuleViolationEvent.EntityID, DEEPLINK_THIS_TIMERANGE); break; default: logger.Warn("Unknown entity type {0} in affectedEntityDefinition in health rule violations", entityType); break; } } healthRuleViolationEvent.ControllerLink = String.Format(DEEPLINK_CONTROLLER, healthRuleViolationEvent.Controller, DEEPLINK_THIS_TIMERANGE); healthRuleViolationEvent.ApplicationLink = String.Format(DEEPLINK_APM_APPLICATION, healthRuleViolationEvent.Controller, healthRuleViolationEvent.ApplicationID, DEEPLINK_THIS_TIMERANGE); healthRuleViolationList.Add(healthRuleViolationEvent); } } loggerConsole.Info("{0} Health Rule Violations", healthRuleViolationList.Count); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + healthRuleViolationList.Count; // Sort them healthRuleViolationList = healthRuleViolationList.OrderBy(o => o.HealthRuleName).ThenBy(o => o.From).ThenBy(o => o.Severity).ToList(); FileIOHelper.WriteListToCSVFile <HealthRuleViolationEvent>(healthRuleViolationList, new HealthRuleViolationEventReportMap(), FilePathMap.ApplicationHealthRuleViolationsIndexFilePath(jobTarget)); #endregion #region Events List <Event> eventsList = new List <Event>(); List <EventDetail> eventDetailsList = new List <EventDetail>(); loggerConsole.Info("Index Events"); foreach (string eventType in EVENT_TYPES) { JArray eventsArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.ApplicationEventsWithDetailsDataFilePath(jobTarget, eventType)); if (eventsArray == null) { eventsArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.ApplicationEventsDataFilePath(jobTarget, eventType)); } if (eventsArray != null) { loggerConsole.Info("{0} Events", eventType); foreach (JObject interestingEventObject in eventsArray) { Event @event = new Event(); @event.Controller = jobTarget.Controller; @event.ApplicationName = jobTarget.Application; @event.ApplicationID = jobTarget.ApplicationID; @event.EventID = getLongValueFromJToken(interestingEventObject, "id"); @event.OccurredUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(interestingEventObject, "eventTime")); @event.Occurred = @event.OccurredUtc.ToLocalTime(); @event.Type = getStringValueFromJToken(interestingEventObject, "type"); @event.SubType = getStringValueFromJToken(interestingEventObject, "subType"); @event.Severity = getStringValueFromJToken(interestingEventObject, "severity"); @event.EventLink = getStringValueFromJToken(interestingEventObject, "deepLinkUrl"); @event.Summary = getStringValueFromJToken(interestingEventObject, "summary"); if (isTokenPropertyNull(interestingEventObject, "triggeredEntity") == false) { @event.TriggeredEntityID = getLongValueFromJToken(interestingEventObject["triggeredEntity"], "entityId"); @event.TriggeredEntityName = getStringValueFromJToken(interestingEventObject["triggeredEntity"], "name"); string entityType = getStringValueFromJToken(interestingEventObject["triggeredEntity"], "entityType"); if (entityTypeStringMapping.ContainsKey(entityType) == true) { @event.TriggeredEntityType = entityTypeStringMapping[entityType]; } else { @event.TriggeredEntityType = entityType; } } foreach (JObject affectedEntity in interestingEventObject["affectedEntities"]) { string entityType = getStringValueFromJToken(affectedEntity, "entityType"); switch (entityType) { case ENTITY_TYPE_FLOWMAP_APPLICATION: // already have this data break; case ENTITY_TYPE_FLOWMAP_TIER: @event.TierID = getIntValueFromJToken(affectedEntity, "entityId"); @event.TierName = getStringValueFromJToken(affectedEntity, "name"); break; case ENTITY_TYPE_FLOWMAP_NODE: @event.NodeID = getIntValueFromJToken(affectedEntity, "entityId"); @event.NodeName = getStringValueFromJToken(affectedEntity, "name"); break; case ENTITY_TYPE_FLOWMAP_MACHINE: @event.MachineID = getIntValueFromJToken(affectedEntity, "entityId"); @event.MachineName = getStringValueFromJToken(affectedEntity, "name"); break; case ENTITY_TYPE_FLOWMAP_BUSINESS_TRANSACTION: @event.BTID = getIntValueFromJToken(affectedEntity, "entityId"); @event.BTName = getStringValueFromJToken(affectedEntity, "name"); break; case ENTITY_TYPE_FLOWMAP_HEALTH_RULE: @event.TriggeredEntityID = getLongValueFromJToken(affectedEntity, "entityId"); @event.TriggeredEntityType = entityTypeStringMapping[getStringValueFromJToken(affectedEntity, "entityType")]; @event.TriggeredEntityName = getStringValueFromJToken(affectedEntity, "name"); break; default: logger.Warn("Unknown entity type {0} in affectedEntities in events", entityType); break; } } @event.ControllerLink = String.Format(DEEPLINK_CONTROLLER, @event.Controller, DEEPLINK_THIS_TIMERANGE); @event.ApplicationLink = String.Format(DEEPLINK_APM_APPLICATION, @event.Controller, @event.ApplicationID, DEEPLINK_THIS_TIMERANGE); if (@event.TierID != 0) { @event.TierLink = String.Format(DEEPLINK_TIER, @event.Controller, @event.ApplicationID, @event.TierID, DEEPLINK_THIS_TIMERANGE); } if (@event.NodeID != 0) { @event.NodeLink = String.Format(DEEPLINK_NODE, @event.Controller, @event.ApplicationID, @event.NodeID, DEEPLINK_THIS_TIMERANGE); } if (@event.BTID != 0) { @event.BTLink = String.Format(DEEPLINK_BUSINESS_TRANSACTION, @event.Controller, @event.ApplicationID, @event.BTID, DEEPLINK_THIS_TIMERANGE); } if (isTokenPropertyNull(interestingEventObject, "details") == false && isTokenPropertyNull(interestingEventObject["details"], "eventDetails") == false) { JArray eventDetailsArray = (JArray)interestingEventObject["details"]["eventDetails"]; if (eventDetailsArray != null) { List <EventDetail> eventDetailsForThisEventList = new List <EventDetail>(eventDetailsArray.Count); foreach (JObject eventDetailObject in eventDetailsArray) { EventDetail eventDetail = new EventDetail(); eventDetail.Controller = @event.Controller; eventDetail.ApplicationName = @event.ApplicationName; eventDetail.ApplicationID = @event.ApplicationID; eventDetail.TierName = @event.TierName; eventDetail.TierID = @event.TierID; eventDetail.NodeName = @event.NodeName; eventDetail.NodeID = @event.NodeID; eventDetail.MachineName = @event.MachineName; eventDetail.MachineID = @event.MachineID; eventDetail.BTName = @event.BTName; eventDetail.BTID = @event.BTID; eventDetail.EventID = @event.EventID; eventDetail.OccurredUtc = @event.OccurredUtc; eventDetail.Occurred = @event.Occurred; eventDetail.Type = @event.Type; eventDetail.SubType = @event.SubType; eventDetail.Severity = @event.Severity; eventDetail.Summary = @event.Summary; eventDetail.DetailName = getStringValueFromJToken(eventDetailObject, "name"); eventDetail.DetailValue = getStringValueFromJToken(eventDetailObject, "value"); // Parse the special types of the event types, such as options and envinronment variable changes bool shouldContinue = true; switch (@event.Type) { #region APPLICATION_CONFIG_CHANGE case "APPLICATION_CONFIG_CHANGE": if (shouldContinue == true) { // Added Option 1 // Removed Option 1 Regex regex = new Regex(@"(.*\sOption)\s\d+", RegexOptions.IgnoreCase); Match match = regex.Match(eventDetail.DetailName); if (match != null && match.Groups.Count == 2) { eventDetail.DetailAction = match.Groups[1].Value; //-Dgw.cc.full.upgrade.intended.date=20191112 //-XX:+UseGCLogFileRotation //-XX:NumberOfGCLogFiles=< number of log files > //-XX:GCLogFileSize=< file size >[ unit ] //-Xloggc:/path/to/gc.log parseJavaStartupOptionIntoEventDetail(eventDetail); shouldContinue = false; } } if (shouldContinue == true) { // Added Variable 1: // Removed Variable 1: Regex regex = new Regex(@"((Added|Removed) Variable)\s\d+", RegexOptions.IgnoreCase); Match match = regex.Match(eventDetail.DetailName); if (match != null && match.Groups.Count == 3) { eventDetail.DetailAction = match.Groups[1].Value; // SSH_TTY=/dev/pts/0 // HOSTNAME=felvqcap1174.farmersinsurance.com parseEnvironmentVariableIntoEventDetail(eventDetail); shouldContinue = false; } } if (shouldContinue == true) { // Modified Variable 1: Regex regex = new Regex(@"(Modified Variable)\s\d+", RegexOptions.IgnoreCase); Match match = regex.Match(eventDetail.DetailName); if (match != null && match.Groups.Count == 2) { eventDetail.DetailAction = match.Groups[1].Value; // MAIL=/var/mail/jbossapp (changed to) MAIL=/var/spool/mail/jbossapp // SHLVL=5 (changed to) SHLVL=3 parseModifiedEnvironmentVariableIntoEventDetail(eventDetail); shouldContinue = false; } } if (shouldContinue == true) { // Modified Property 2: Regex regex = new Regex(@"(Modified Property)\s\d+", RegexOptions.IgnoreCase); Match match = regex.Match(eventDetail.DetailName); if (match != null && match.Groups.Count == 2) { eventDetail.DetailAction = match.Groups[1].Value; // gw.cc.full.upgrade.intended.date=20191112 (changed to) gw.cc.full.upgrade.intended.date=20191113 // user.dir=/jboss/scripts/jenkins/28/slave/workspace/ClaimsCenter/GWCC_GW_DB_APP_DP/perfcc06post (changed to) user.dir=/jboss/scripts/ccperf06 parseModifiedPropertyIntoEventDetail(eventDetail); shouldContinue = false; } } break; #endregion #region POLICY_*** case "POLICY_OPEN_WARNING": case "POLICY_OPEN_CRITICAL": case "POLICY_CLOSE_WARNING": case "POLICY_CLOSE_CRITICAL": case "POLICY_UPGRADED": case "POLICY_DOWNGRADED": case "POLICY_CANCELED_WARNING": case "POLICY_CANCELED_CRITICAL": case "POLICY_CONTINUES_CRITICAL": case "POLICY_CONTINUES_WARNING": if (eventDetail.DetailName == "Evaluation End Time" || eventDetail.DetailName == "Evaluation Start Time") { // These are a unix datetime value DateTime dateTimeVal1 = UnixTimeHelper.ConvertFromUnixTimestamp(Convert.ToInt64(eventDetail.DetailValue)); eventDetail.DataType = "DateTime"; eventDetail.DetailValue = dateTimeVal1.ToLocalTime().ToString("G"); } break; #endregion default: break; } if (eventDetail.DataType == null || eventDetail.DataType.Length == 0) { // Get datatype of value long longVal = 0; double doubleVal = 0; DateTime dateTimeVal = DateTime.MinValue; if (Int64.TryParse(eventDetail.DetailValue, out longVal) == true) { eventDetail.DataType = "Integer"; } else if (Double.TryParse(eventDetail.DetailValue, out doubleVal) == true) { eventDetail.DataType = "Double"; } else if (DateTime.TryParse(eventDetail.DetailValue, out dateTimeVal) == true) { eventDetail.DataType = "DateTime"; } else { eventDetail.DataType = "String"; } } eventDetailsForThisEventList.Add(eventDetail); } @event.NumDetails = eventDetailsForThisEventList.Count; // Sort them eventDetailsForThisEventList = eventDetailsForThisEventList.OrderBy(o => o.DetailName).ToList(); eventDetailsList.AddRange(eventDetailsForThisEventList); } } eventsList.Add(@event); } } } loggerConsole.Info("{0} Events", eventsList.Count); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + eventsList.Count; // Sort them eventsList = eventsList.OrderBy(o => o.Type).ThenBy(o => o.Occurred).ThenBy(o => o.Severity).ToList(); FileIOHelper.WriteListToCSVFile <Event>(eventsList, new EventReportMap(), FilePathMap.ApplicationEventsIndexFilePath(jobTarget)); FileIOHelper.WriteListToCSVFile <EventDetail>(eventDetailsList, new EventDetailReportMap(), FilePathMap.ApplicationEventDetailsIndexFilePath(jobTarget)); #endregion #region Application ApplicationEventSummary application = new ApplicationEventSummary(); application.Controller = jobTarget.Controller; application.ApplicationName = jobTarget.Application; application.ApplicationID = jobTarget.ApplicationID; application.Type = jobTarget.Type; application.NumEvents = eventsList.Count; application.NumEventsError = eventsList.Count(e => e.Severity == "ERROR"); application.NumEventsWarning = eventsList.Count(e => e.Severity == "WARN"); application.NumEventsInfo = eventsList.Count(e => e.Severity == "INFO"); application.NumHRViolations = healthRuleViolationList.Count; application.NumHRViolationsCritical = healthRuleViolationList.Count(e => e.Severity == "CRITICAL"); application.NumHRViolationsWarning = healthRuleViolationList.Count(e => e.Severity == "WARNING"); application.Duration = (int)(jobConfiguration.Input.TimeRange.To - jobConfiguration.Input.TimeRange.From).Duration().TotalMinutes; application.From = jobConfiguration.Input.TimeRange.From.ToLocalTime(); application.To = jobConfiguration.Input.TimeRange.To.ToLocalTime(); application.FromUtc = jobConfiguration.Input.TimeRange.From; application.ToUtc = jobConfiguration.Input.TimeRange.To; // Determine what kind of entity we are dealing with and adjust accordingly application.ControllerLink = String.Format(DEEPLINK_CONTROLLER, application.Controller, DEEPLINK_THIS_TIMERANGE); application.ApplicationLink = String.Format(DEEPLINK_APM_APPLICATION, application.Controller, application.ApplicationID, DEEPLINK_THIS_TIMERANGE); if (application.NumEvents > 0 || application.NumHRViolations > 0) { application.HasActivity = true; } List <ApplicationEventSummary> applicationList = new List <ApplicationEventSummary>(1); applicationList.Add(application); FileIOHelper.WriteListToCSVFile(applicationList, new ApplicationEventSummaryReportMap(), FilePathMap.ApplicationEventsSummaryIndexFilePath(jobTarget)); #endregion #region Combine All for Report CSV // If it is the first one, clear out the combined folder if (reportFolderCleaned == false) { FileIOHelper.DeleteFolder(FilePathMap.ApplicationEventsReportFolderPath()); Thread.Sleep(1000); FileIOHelper.CreateFolder(FilePathMap.ApplicationEventsReportFolderPath()); reportFolderCleaned = true; } // Append all the individual report files into one if (File.Exists(FilePathMap.ApplicationHealthRuleViolationsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.ApplicationHealthRuleViolationsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.ApplicationHealthRuleViolationsReportFilePath(), FilePathMap.ApplicationHealthRuleViolationsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.ApplicationEventsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.ApplicationEventsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.ApplicationEventsReportFilePath(), FilePathMap.ApplicationEventsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.ApplicationEventDetailsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.ApplicationEventDetailsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.ApplicationEventDetailsReportFilePath(), FilePathMap.ApplicationEventDetailsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.ApplicationEventsSummaryIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.ApplicationEventsSummaryIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.ApplicationEventsSummaryReportFilePath(), FilePathMap.ApplicationEventsSummaryIndexFilePath(jobTarget)); } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(jobConfiguration) == false) { return(true); } List <JobTarget> listOfTargetsAlreadyProcessed = new List <JobTarget>(jobConfiguration.Target.Count); bool reportFolderCleaned = false; bool haveProcessedAtLeastOneDBCollector = false; // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); if (listOfTargetsAlreadyProcessed.Count(j => (j.Controller == jobTarget.Controller) && (j.ApplicationID == jobTarget.ApplicationID)) > 0) { // Already saw this target, like an APM and WEB pairs together continue; } // For databases, we only process this once for the first collector we've seen if (jobTarget.Type == APPLICATION_TYPE_DB) { if (haveProcessedAtLeastOneDBCollector == false) { haveProcessedAtLeastOneDBCollector = true; } else { continue; } } listOfTargetsAlreadyProcessed.Add(jobTarget); #region Prepare time variables long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From); long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To); long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000); string DEEPLINK_THIS_TIMERANGE = String.Format(DEEPLINK_TIMERANGE_BETWEEN_TIMES, toTimeUnix, fromTimeUnix, differenceInMinutes); #endregion #region Health Rule violations List <HealthRuleViolationEvent> healthRuleViolationList = new List <HealthRuleViolationEvent>(); loggerConsole.Info("Index Health Rule Violations"); JArray healthRuleViolationsArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.ApplicationHealthRuleViolationsDataFilePath(jobTarget)); if (healthRuleViolationsArray != null) { foreach (JObject interestingEventObject in healthRuleViolationsArray) { HealthRuleViolationEvent healthRuleViolationEvent = new HealthRuleViolationEvent(); healthRuleViolationEvent.Controller = jobTarget.Controller; healthRuleViolationEvent.ApplicationName = jobTarget.Application; healthRuleViolationEvent.ApplicationID = jobTarget.ApplicationID; healthRuleViolationEvent.EventID = getLongValueFromJToken(interestingEventObject, "id"); healthRuleViolationEvent.FromUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(interestingEventObject, "startTimeInMillis")); healthRuleViolationEvent.From = healthRuleViolationEvent.FromUtc.ToLocalTime(); if (getLongValueFromJToken(interestingEventObject, "endTimeInMillis") > 0) { healthRuleViolationEvent.ToUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(interestingEventObject, "endTimeInMillis")); healthRuleViolationEvent.To = healthRuleViolationEvent.FromUtc.ToLocalTime(); } healthRuleViolationEvent.Status = getStringValueFromJToken(interestingEventObject, "incidentStatus"); healthRuleViolationEvent.Severity = getStringValueFromJToken(interestingEventObject, "severity"); healthRuleViolationEvent.EventLink = String.Format(DEEPLINK_INCIDENT, healthRuleViolationEvent.Controller, healthRuleViolationEvent.ApplicationID, healthRuleViolationEvent.EventID, getStringValueFromJToken(interestingEventObject, "startTimeInMillis"), DEEPLINK_THIS_TIMERANGE);; healthRuleViolationEvent.Description = getStringValueFromJToken(interestingEventObject, "description"); if (isTokenPropertyNull(interestingEventObject, "triggeredEntityDefinition") == false) { healthRuleViolationEvent.HealthRuleID = getLongValueFromJToken(interestingEventObject["triggeredEntityDefinition"], "entityId"); healthRuleViolationEvent.HealthRuleName = getStringValueFromJToken(interestingEventObject["triggeredEntityDefinition"], "name"); // TODO the health rule can't be hotlinked to until platform rewrites the screen that opens from Flash healthRuleViolationEvent.HealthRuleLink = String.Format(DEEPLINK_HEALTH_RULE, healthRuleViolationEvent.Controller, healthRuleViolationEvent.ApplicationID, healthRuleViolationEvent.HealthRuleID, DEEPLINK_THIS_TIMERANGE); } if (isTokenPropertyNull(interestingEventObject, "affectedEntityDefinition") == false) { healthRuleViolationEvent.EntityID = getIntValueFromJToken(interestingEventObject["affectedEntityDefinition"], "entityId"); healthRuleViolationEvent.EntityName = getStringValueFromJToken(interestingEventObject["affectedEntityDefinition"], "name"); string entityType = getStringValueFromJToken(interestingEventObject["affectedEntityDefinition"], "entityType"); if (entityTypeStringMapping.ContainsKey(entityType) == true) { healthRuleViolationEvent.EntityType = entityTypeStringMapping[entityType]; } else { healthRuleViolationEvent.EntityType = entityType; } // Come up with links switch (entityType) { case ENTITY_TYPE_FLOWMAP_APPLICATION: healthRuleViolationEvent.EntityLink = String.Format(DEEPLINK_APM_APPLICATION, healthRuleViolationEvent.Controller, healthRuleViolationEvent.ApplicationID, DEEPLINK_THIS_TIMERANGE); break; case ENTITY_TYPE_FLOWMAP_APPLICATION_MOBILE: healthRuleViolationEvent.EntityLink = String.Format(DEEPLINK_APPLICATION_MOBILE, healthRuleViolationEvent.Controller, healthRuleViolationEvent.ApplicationID, healthRuleViolationEvent.EntityID, DEEPLINK_THIS_TIMERANGE); break; case ENTITY_TYPE_FLOWMAP_TIER: healthRuleViolationEvent.EntityLink = String.Format(DEEPLINK_TIER, healthRuleViolationEvent.Controller, healthRuleViolationEvent.ApplicationID, healthRuleViolationEvent.EntityID, DEEPLINK_THIS_TIMERANGE); break; case ENTITY_TYPE_FLOWMAP_NODE: healthRuleViolationEvent.EntityLink = String.Format(DEEPLINK_NODE, healthRuleViolationEvent.Controller, healthRuleViolationEvent.ApplicationID, healthRuleViolationEvent.EntityID, DEEPLINK_THIS_TIMERANGE); break; case ENTITY_TYPE_FLOWMAP_BUSINESS_TRANSACTION: healthRuleViolationEvent.EntityLink = String.Format(DEEPLINK_BUSINESS_TRANSACTION, healthRuleViolationEvent.Controller, healthRuleViolationEvent.ApplicationID, healthRuleViolationEvent.EntityID, DEEPLINK_THIS_TIMERANGE); break; case ENTITY_TYPE_FLOWMAP_BACKEND: healthRuleViolationEvent.EntityLink = String.Format(DEEPLINK_BACKEND, healthRuleViolationEvent.Controller, healthRuleViolationEvent.ApplicationID, healthRuleViolationEvent.EntityID, DEEPLINK_THIS_TIMERANGE); break; default: logger.Warn("Unknown entity type {0} in affectedEntityDefinition in health rule violations", entityType); break; } } healthRuleViolationEvent.ControllerLink = String.Format(DEEPLINK_CONTROLLER, healthRuleViolationEvent.Controller, DEEPLINK_THIS_TIMERANGE); healthRuleViolationEvent.ApplicationLink = String.Format(DEEPLINK_APM_APPLICATION, healthRuleViolationEvent.Controller, healthRuleViolationEvent.ApplicationID, DEEPLINK_THIS_TIMERANGE); healthRuleViolationList.Add(healthRuleViolationEvent); } } loggerConsole.Info("{0} Health Rule Violations", healthRuleViolationList.Count); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + healthRuleViolationList.Count; // Sort them healthRuleViolationList = healthRuleViolationList.OrderBy(o => o.HealthRuleName).ThenBy(o => o.From).ThenBy(o => o.Severity).ToList(); FileIOHelper.WriteListToCSVFile <HealthRuleViolationEvent>(healthRuleViolationList, new HealthRuleViolationEventReportMap(), FilePathMap.ApplicationHealthRuleViolationsIndexFilePath(jobTarget)); #endregion #region Events List <Event> eventsList = new List <Event>(); loggerConsole.Info("Index Events"); foreach (string eventType in EVENT_TYPES) { JArray eventsArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.ApplicationEventsDataFilePath(jobTarget, eventType)); if (eventsArray != null) { loggerConsole.Info("{0} Events", eventType); foreach (JObject interestingEventObject in eventsArray) { Event @event = new Event(); @event.Controller = jobTarget.Controller; @event.ApplicationName = jobTarget.Application; @event.ApplicationID = jobTarget.ApplicationID; @event.EventID = getLongValueFromJToken(interestingEventObject, "id"); @event.OccurredUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(interestingEventObject, "eventTime")); @event.Occurred = @event.OccurredUtc.ToLocalTime(); @event.Type = getStringValueFromJToken(interestingEventObject, "type"); @event.SubType = getStringValueFromJToken(interestingEventObject, "subType"); @event.Severity = getStringValueFromJToken(interestingEventObject, "severity"); @event.EventLink = getStringValueFromJToken(interestingEventObject, "deepLinkUrl"); @event.Summary = getStringValueFromJToken(interestingEventObject, "summary"); if (isTokenPropertyNull(interestingEventObject, "triggeredEntity") == false) { @event.TriggeredEntityID = getLongValueFromJToken(interestingEventObject["triggeredEntity"], "entityId"); @event.TriggeredEntityName = getStringValueFromJToken(interestingEventObject["triggeredEntity"], "name"); string entityType = getStringValueFromJToken(interestingEventObject["triggeredEntity"], "entityType"); if (entityTypeStringMapping.ContainsKey(entityType) == true) { @event.TriggeredEntityType = entityTypeStringMapping[entityType]; } else { @event.TriggeredEntityType = entityType; } } foreach (JObject affectedEntity in interestingEventObject["affectedEntities"]) { string entityType = getStringValueFromJToken(affectedEntity, "entityType"); switch (entityType) { case ENTITY_TYPE_FLOWMAP_APPLICATION: // already have this data break; case ENTITY_TYPE_FLOWMAP_TIER: @event.TierID = getIntValueFromJToken(affectedEntity, "entityId"); @event.TierName = getStringValueFromJToken(affectedEntity, "name"); break; case ENTITY_TYPE_FLOWMAP_NODE: @event.NodeID = getIntValueFromJToken(affectedEntity, "entityId"); @event.NodeName = getStringValueFromJToken(affectedEntity, "name"); break; case ENTITY_TYPE_FLOWMAP_MACHINE: @event.MachineID = getIntValueFromJToken(affectedEntity, "entityId"); @event.MachineName = getStringValueFromJToken(affectedEntity, "name"); break; case ENTITY_TYPE_FLOWMAP_BUSINESS_TRANSACTION: @event.BTID = getIntValueFromJToken(affectedEntity, "entityId"); @event.BTName = getStringValueFromJToken(affectedEntity, "name"); break; case ENTITY_TYPE_FLOWMAP_HEALTH_RULE: @event.TriggeredEntityID = getLongValueFromJToken(affectedEntity, "entityId"); @event.TriggeredEntityType = entityTypeStringMapping[getStringValueFromJToken(affectedEntity, "entityType")]; @event.TriggeredEntityName = getStringValueFromJToken(affectedEntity, "name"); break; default: logger.Warn("Unknown entity type {0} in affectedEntities in events", entityType); break; } } @event.ControllerLink = String.Format(DEEPLINK_CONTROLLER, @event.Controller, DEEPLINK_THIS_TIMERANGE); @event.ApplicationLink = String.Format(DEEPLINK_APM_APPLICATION, @event.Controller, @event.ApplicationID, DEEPLINK_THIS_TIMERANGE); if (@event.TierID != 0) { @event.TierLink = String.Format(DEEPLINK_TIER, @event.Controller, @event.ApplicationID, @event.TierID, DEEPLINK_THIS_TIMERANGE); } if (@event.NodeID != 0) { @event.NodeLink = String.Format(DEEPLINK_NODE, @event.Controller, @event.ApplicationID, @event.NodeID, DEEPLINK_THIS_TIMERANGE); } if (@event.BTID != 0) { @event.BTLink = String.Format(DEEPLINK_BUSINESS_TRANSACTION, @event.Controller, @event.ApplicationID, @event.BTID, DEEPLINK_THIS_TIMERANGE); } eventsList.Add(@event); } } } loggerConsole.Info("{0} Events", eventsList.Count); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + eventsList.Count; // Sort them eventsList = eventsList.OrderBy(o => o.Type).ThenBy(o => o.Occurred).ThenBy(o => o.Severity).ToList(); FileIOHelper.WriteListToCSVFile <Event>(eventsList, new EventReportMap(), FilePathMap.ApplicationEventsIndexFilePath(jobTarget)); #endregion #region Application ApplicationEventSummary application = new ApplicationEventSummary(); application.Controller = jobTarget.Controller; application.ApplicationName = jobTarget.Application; application.ApplicationID = jobTarget.ApplicationID; application.Type = jobTarget.Type; application.NumEvents = eventsList.Count; application.NumEventsError = eventsList.Count(e => e.Severity == "ERROR"); application.NumEventsWarning = eventsList.Count(e => e.Severity == "WARN"); application.NumEventsInfo = eventsList.Count(e => e.Severity == "INFO"); application.NumHRViolations = healthRuleViolationList.Count; application.NumHRViolationsCritical = healthRuleViolationList.Count(e => e.Severity == "CRITICAL"); application.NumHRViolationsWarning = healthRuleViolationList.Count(e => e.Severity == "WARNING"); application.Duration = (int)(jobConfiguration.Input.TimeRange.To - jobConfiguration.Input.TimeRange.From).Duration().TotalMinutes; application.From = jobConfiguration.Input.TimeRange.From.ToLocalTime(); application.To = jobConfiguration.Input.TimeRange.To.ToLocalTime(); application.FromUtc = jobConfiguration.Input.TimeRange.From; application.ToUtc = jobConfiguration.Input.TimeRange.To; // Determine what kind of entity we are dealing with and adjust accordingly application.ControllerLink = String.Format(DEEPLINK_CONTROLLER, application.Controller, DEEPLINK_THIS_TIMERANGE); application.ApplicationLink = String.Format(DEEPLINK_APM_APPLICATION, application.Controller, application.ApplicationID, DEEPLINK_THIS_TIMERANGE); if (application.NumEvents > 0 || application.NumHRViolations > 0) { application.HasActivity = true; } List <ApplicationEventSummary> applicationList = new List <ApplicationEventSummary>(1); applicationList.Add(application); FileIOHelper.WriteListToCSVFile(applicationList, new ApplicationEventSummaryReportMap(), FilePathMap.ApplicationEventsSummaryIndexFilePath(jobTarget)); #endregion #region Combine All for Report CSV // If it is the first one, clear out the combined folder if (reportFolderCleaned == false) { FileIOHelper.DeleteFolder(FilePathMap.ApplicationEventsReportFolderPath()); Thread.Sleep(1000); FileIOHelper.CreateFolder(FilePathMap.ApplicationEventsReportFolderPath()); reportFolderCleaned = true; } // Append all the individual report files into one if (File.Exists(FilePathMap.ApplicationHealthRuleViolationsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.ApplicationHealthRuleViolationsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.ApplicationHealthRuleViolationsReportFilePath(), FilePathMap.ApplicationHealthRuleViolationsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.ApplicationEventsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.ApplicationEventsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.ApplicationEventsReportFilePath(), FilePathMap.ApplicationEventsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.ApplicationEventsSummaryIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.ApplicationEventsSummaryIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.ApplicationEventsSummaryReportFilePath(), FilePathMap.ApplicationEventsSummaryIndexFilePath(jobTarget)); } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(jobConfiguration) == false) { return(true); } bool reportFolderCleaned = false; // Process each Controller once int i = 0; var controllers = jobConfiguration.Target.GroupBy(t => t.Controller); foreach (var controllerGroup in controllers) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = controllerGroup.ToList()[0]; StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; stepTimingTarget.NumEntities = 0; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Dashboards List and Widgets List loggerConsole.Info("Dashboards and their widgets"); List <Dashboard> dashboardsList = new List <Dashboard>(1024); List <DashboardWidget> dashboardWidgetsAllList = new List <DashboardWidget>(10240); List <DashboardMetricSeries> dashboardMetricSeriesAllList = new List <DashboardMetricSeries>(10240); JArray dashboardsArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.ControllerDashboards(jobTarget)); if (dashboardsArray != null) { int j = 0; foreach (JObject dashboardObject in dashboardsArray) { Dashboard dashboard = new Dashboard(); dashboard.Controller = jobTarget.Controller; dashboard.DashboardName = getStringValueFromJToken(dashboardObject, "name"); dashboard.Description = getStringValueFromJToken(dashboardObject, "description"); dashboard.CanvasType = getStringValueFromJToken(dashboardObject, "canvasType").Replace("CANVAS_TYPE_", ""); dashboard.TemplateEntityType = getStringValueFromJToken(dashboardObject, "templateEntityType"); dashboard.SecurityToken = getStringValueFromJToken(dashboardObject, "securityToken"); if (dashboard.SecurityToken.Length > 0) { dashboard.IsShared = true; } dashboard.IsSharingRevoked = getBoolValueFromJToken(dashboardObject, "sharingRevoked"); dashboard.IsTemplate = getBoolValueFromJToken(dashboardObject, "template"); dashboard.Height = getIntValueFromJToken(dashboardObject, "height"); dashboard.Width = getIntValueFromJToken(dashboardObject, "width"); dashboard.BackgroundColor = getIntValueFromJToken(dashboardObject, "backgroundColor").ToString("X6"); dashboard.MinutesBefore = getIntValueFromJToken(dashboardObject, "minutesBeforeAnchorTime"); dashboard.RefreshInterval = getIntValueFromJToken(dashboardObject, "refreshInterval") / 1000; dashboard.StartTimeUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(dashboardObject, "startTime")); try { dashboard.StartTime = dashboard.StartTimeUtc.ToLocalTime(); } catch { } dashboard.EndTimeUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(dashboardObject, "endTime")); try { dashboard.EndTime = dashboard.EndTimeUtc.ToLocalTime(); } catch { } dashboard.CreatedBy = getStringValueFromJToken(dashboardObject, "createdBy"); dashboard.CreatedOnUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(dashboardObject, "createdOn")); try { dashboard.CreatedOn = dashboard.CreatedOnUtc.ToLocalTime(); } catch { } dashboard.UpdatedBy = getStringValueFromJToken(dashboardObject, "modifiedBy"); dashboard.UpdatedOnUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(dashboardObject, "modifiedOn")); try { dashboard.UpdatedOn = dashboard.UpdatedOnUtc.ToLocalTime(); } catch { } dashboard.DashboardID = getLongValueFromJToken(dashboardObject, "id"); dashboard.DashboardLink = String.Format(DEEPLINK_DASHBOARD, dashboard.Controller, dashboard.DashboardID, DEEPLINK_TIMERANGE_LAST_15_MINUTES); // Now parse the Widgets JObject dashboardDetailObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.ControllerDashboard(jobTarget, dashboard.DashboardName, dashboard.DashboardID)); if (dashboardDetailObject != null && isTokenPropertyNull(dashboardDetailObject, "widgetTemplates") == false) { List <DashboardWidget> dashboardWidgetsList = new List <DashboardWidget>(dashboardDetailObject["widgetTemplates"].Count()); int dashboardWidgetIndex = 0; foreach (JObject dashboardWidgetObject in dashboardDetailObject["widgetTemplates"]) { DashboardWidget dashboardWidget = new DashboardWidget(); dashboardWidget.Controller = dashboard.Controller; dashboardWidget.DashboardName = dashboard.DashboardName; dashboardWidget.DashboardID = dashboard.DashboardID; dashboardWidget.CanvasType = dashboard.CanvasType; dashboardWidget.WidgetType = getStringValueFromJToken(dashboardWidgetObject, "widgetType"); dashboardWidget.Index = dashboardWidgetIndex; dashboardWidget.ApplicationName = getStringValueFromJToken(dashboardWidgetObject["applicationReference"], "applicationName"); dashboardWidget.EntityType = getStringValueFromJToken(dashboardWidgetObject, "entityType"); dashboardWidget.EntitySelectionType = getStringValueFromJToken(dashboardWidgetObject, "entitySelectionType"); try { if (isTokenPropertyNull(dashboardWidgetObject, "entityReferences") == false) { string[] entities = new string[dashboardWidgetObject["entityReferences"].Count()]; int entitiesIndex = 0; foreach (JToken entityReferenceToken in dashboardWidgetObject["entityReferences"]) { string scopingEntityName = getStringValueFromJToken(entityReferenceToken, "scopingEntityName"); string entityName = getStringValueFromJToken(entityReferenceToken, "entityName"); string subType = getStringValueFromJToken(entityReferenceToken, "subtype"); StringBuilder sb = new StringBuilder(100); sb.Append(scopingEntityName); if (scopingEntityName.Length > 0) { sb.Append("/"); } sb.Append(entityName); if (subType.Length > 0) { sb.AppendFormat(" [{0}]", subType); } entities[entitiesIndex] = sb.ToString(); entitiesIndex++;; } dashboardWidget.SelectedEntities = String.Join(";", entities); dashboardWidget.NumSelectedEntities = entities.Length; } } catch { } dashboardWidget.Title = getStringValueFromJToken(dashboardWidgetObject, "title"); dashboardWidget.Description = getStringValueFromJToken(dashboardWidgetObject, "description"); dashboardWidget.Label = getStringValueFromJToken(dashboardWidgetObject, "label"); dashboardWidget.Text = getStringValueFromJToken(dashboardWidgetObject, "text"); dashboardWidget.TextAlign = getStringValueFromJToken(dashboardWidgetObject, "textAlign"); dashboardWidget.Width = getIntValueFromJToken(dashboardWidgetObject, "width"); dashboardWidget.Height = getIntValueFromJToken(dashboardWidgetObject, "height"); dashboardWidget.MinWidth = getIntValueFromJToken(dashboardWidgetObject, "minHeight"); dashboardWidget.MinHeight = getIntValueFromJToken(dashboardWidgetObject, "minWidth"); dashboardWidget.X = getIntValueFromJToken(dashboardWidgetObject, "x"); dashboardWidget.Y = getIntValueFromJToken(dashboardWidgetObject, "y"); dashboardWidget.ForegroundColor = getIntValueFromJToken(dashboardWidgetObject, "color").ToString("X6"); dashboardWidget.BackgroundColor = getIntValueFromJToken(dashboardWidgetObject, "backgroundColor").ToString("X6"); dashboardWidget.BackgroundAlpha = getDoubleValueFromJToken(dashboardWidgetObject, "backgroundAlpha"); dashboardWidget.BorderColor = getIntValueFromJToken(dashboardWidgetObject, "borderColor").ToString("X6"); dashboardWidget.BorderSize = getIntValueFromJToken(dashboardWidgetObject, "borderThickness"); dashboardWidget.IsBorderEnabled = getBoolValueFromJToken(dashboardWidgetObject, "borderEnabled"); dashboardWidget.Margin = getIntValueFromJToken(dashboardWidgetObject, "margin"); try { dashboardWidget.NumDataSeries = dashboardWidgetObject["dataSeriesTemplates"].Count(); } catch { } dashboardWidget.FontSize = getIntValueFromJToken(dashboardWidgetObject, "fontSize"); dashboardWidget.MinutesBeforeAnchor = getIntValueFromJToken(dashboardWidgetObject, "minutesBeforeAnchorTime"); dashboardWidget.VerticalAxisLabel = getStringValueFromJToken(dashboardWidgetObject, "verticalAxisLabel"); dashboardWidget.HorizontalAxisLabel = getStringValueFromJToken(dashboardWidgetObject, "horizontalAxisLabel"); dashboardWidget.AxisType = getStringValueFromJToken(dashboardWidgetObject, "axisType"); dashboardWidget.IsMultipleYAxis = getBoolValueFromJToken(dashboardWidgetObject, "multipleYAxis"); dashboardWidget.StackMode = getStringValueFromJToken(dashboardWidgetObject, "stackMode"); dashboardWidget.AggregationType = getStringValueFromJToken(dashboardWidgetObject, "aggregationType"); dashboardWidget.DrillDownURL = getStringValueFromJToken(dashboardWidgetObject, "drillDownUrl"); dashboardWidget.IsDrillDownMetricBrowser = getBoolValueFromJToken(dashboardWidgetObject, "useMetricBrowserAsDrillDown"); dashboardWidget.IsShowEvents = getBoolValueFromJToken(dashboardWidgetObject, "showEvents"); dashboardWidget.EventFilter = getStringValueOfObjectFromJToken(dashboardWidgetObject, "eventFilterTemplate", false); dashboardWidget.ImageURL = getStringValueFromJToken(dashboardWidgetObject, "imageURL"); if (dashboardWidget.ImageURL.Length > 0) { if (dashboardWidget.ImageURL.StartsWith("data:") == true) { dashboardWidget.EmbeddedImageSize = dashboardWidget.ImageURL.Length; dashboardWidget.ImageURL = "Embedded base64 image"; } } dashboardWidget.SourceURL = getStringValueFromJToken(dashboardWidgetObject, "sourceURL"); dashboardWidget.IsSandbox = getBoolValueFromJToken(dashboardWidgetObject, "sandbox"); if (dashboardWidget.WidgetType == "AnalyticsWidget") { try { if (dashboardWidgetObject["adqlQueryList"].Count() == 0) { dashboardWidget.AnalyticsQueries = String.Empty; } else if (dashboardWidgetObject["adqlQueryList"].Count() == 1) { dashboardWidget.AnalyticsQueries = dashboardWidgetObject["adqlQueryList"][0].ToString(); } else { dashboardWidget.AnalyticsQueries = getStringValueOfObjectFromJToken(dashboardWidgetObject, "adqlQueryList", false); } } catch { } dashboardWidget.AnalyticsWidgetType = getStringValueFromJToken(dashboardWidgetObject, "analyticsWidgetType"); dashboardWidget.AnalyticsSearchMode = getStringValueFromJToken(dashboardWidgetObject, "searchMode"); } #region Maybe discern between Widget Types? //switch (dashboardWidget.WidgetType) //{ // case "HealthListWidget": // break; // case "ImageWidget": // break; // case "TextWidget": // break; // case "GraphWidget": // break; // case "MetricLabelWidget": // break; // case "EventListWidget": // break; // case "AnalyticsWidget": // break; // case "PieWidget": // break; // case "GaugeWidget": // break; // case "IFrameWidget": // break; // default: // logger.Warn("Unknown Widget Type {0} in {1}, Widget {2}", dashboardWidget.WidgetType, dashboard, dashboardWidget.Index); // loggerConsole.Warn("Unknown Widget Type {0} in {1}, Widget {2}", dashboardWidget.WidgetType, dashboard, dashboardWidget.Index); // break; //} #endregion dashboardWidgetsList.Add(dashboardWidget); // Now process metric data series for widgets that support them if (dashboardWidget.NumDataSeries > 0) { List <DashboardMetricSeries> dashboardMetricSeriesList = new List <DashboardMetricSeries>(dashboardWidget.NumDataSeries); foreach (JObject dashboardMetricSeriesObject in dashboardWidgetObject["dataSeriesTemplates"]) { DashboardMetricSeries dashboardMetricSeries = new DashboardMetricSeries(); dashboardMetricSeries.Controller = dashboard.Controller; dashboardMetricSeries.DashboardName = dashboard.DashboardName; dashboardMetricSeries.DashboardID = dashboard.DashboardID; dashboardMetricSeries.CanvasType = dashboard.CanvasType; dashboardMetricSeries.WidgetType = dashboardWidget.WidgetType; dashboardMetricSeries.Index = dashboardWidget.Index; dashboardMetricSeries.SeriesName = getStringValueFromJToken(dashboardMetricSeriesObject, "name"); dashboardMetricSeries.SeriesType = getStringValueFromJToken(dashboardMetricSeriesObject, "seriesType"); dashboardMetricSeries.MetricType = getStringValueFromJToken(dashboardMetricSeriesObject, "metricType"); if (isTokenPropertyNull(dashboardMetricSeriesObject, "colorPalette") == false) { if (isTokenPropertyNull(dashboardMetricSeriesObject["colorPalette"], "colors") == false) { string[] entities = new string[dashboardMetricSeriesObject["colorPalette"]["colors"].Count()]; int entitiesIndex = 0; foreach (JToken entityReferenceToken in dashboardMetricSeriesObject["colorPalette"]["colors"]) { int color = (int)entityReferenceToken; entities[entitiesIndex] = color.ToString("X6"); entitiesIndex++;; } dashboardMetricSeries.Colors = String.Join(";", entities); dashboardMetricSeries.NumColors = entities.Length; } } dashboardMetricSeries.Axis = getStringValueFromJToken(dashboardMetricSeriesObject, "axisPosition"); if (isTokenPropertyNull(dashboardMetricSeriesObject, "metricMatchCriteriaTemplate") == false) { JObject dashboardMetricSeriesMetricMatchCriteriaTemplateObject = (JObject)dashboardMetricSeriesObject["metricMatchCriteriaTemplate"]; dashboardMetricSeries.MaxResults = getIntValueFromJToken(dashboardMetricSeriesMetricMatchCriteriaTemplateObject, "maxResults"); dashboardMetricSeries.ApplicationName = getStringValueFromJToken(dashboardMetricSeriesMetricMatchCriteriaTemplateObject, "applicationName"); dashboardMetricSeries.Expression = getStringValueFromJToken(dashboardMetricSeriesMetricMatchCriteriaTemplateObject, "expressionString"); dashboardMetricSeries.EvalScopeType = getStringValueFromJToken(dashboardMetricSeriesMetricMatchCriteriaTemplateObject, "evaluationScopeType"); dashboardMetricSeries.Baseline = getStringValueFromJToken(dashboardMetricSeriesMetricMatchCriteriaTemplateObject, "baselineName"); dashboardMetricSeries.DisplayStyle = getStringValueFromJToken(dashboardMetricSeriesMetricMatchCriteriaTemplateObject, "metricDisplayNameStyle"); dashboardMetricSeries.DisplayFormat = getStringValueFromJToken(dashboardMetricSeriesMetricMatchCriteriaTemplateObject, "metricDisplayNameCustomFormat"); dashboardMetricSeries.IsRollup = getBoolValueFromJToken(dashboardMetricSeriesMetricMatchCriteriaTemplateObject, "rollupMetricData"); dashboardMetricSeries.UseActiveBaseline = getBoolValueFromJToken(dashboardMetricSeriesMetricMatchCriteriaTemplateObject, "useActiveBaseline"); if (getBoolValueFromJToken(dashboardMetricSeriesMetricMatchCriteriaTemplateObject, "sortResultsAscending") == true) { dashboardMetricSeries.SortDirection = "Ascending"; } else { dashboardMetricSeries.SortDirection = "Descending"; } if (isTokenPropertyNull(dashboardMetricSeriesMetricMatchCriteriaTemplateObject, "entityMatchCriteria") == false) { JObject dashboardMetricSeriesEntityMatchCriteriaObject = (JObject)dashboardMetricSeriesMetricMatchCriteriaTemplateObject["entityMatchCriteria"]; dashboardMetricSeries.IsSummary = getBoolValueFromJToken(dashboardMetricSeriesEntityMatchCriteriaObject, "summary"); dashboardMetricSeries.EntityType = getStringValueFromJToken(dashboardMetricSeriesEntityMatchCriteriaObject, "entityType"); dashboardMetricSeries.EntitySelectionType = getStringValueFromJToken(dashboardMetricSeriesEntityMatchCriteriaObject, "matchCriteriaType"); dashboardMetricSeries.AgentType = getStringValueOfObjectFromJToken(dashboardMetricSeriesEntityMatchCriteriaObject, "agentTypes", true); if (isTokenPropertyNull(dashboardMetricSeriesEntityMatchCriteriaObject, "entityNames") == false) { string[] entities = new string[dashboardMetricSeriesEntityMatchCriteriaObject["entityNames"].Count()]; int entitiesIndex = 0; foreach (JToken entityReferenceToken in dashboardMetricSeriesEntityMatchCriteriaObject["entityNames"]) { string scopingEntityName = getStringValueFromJToken(entityReferenceToken, "scopingEntityName"); string entityName = getStringValueFromJToken(entityReferenceToken, "entityName"); string subType = getStringValueFromJToken(entityReferenceToken, "subtype"); StringBuilder sb = new StringBuilder(100); sb.Append(scopingEntityName); if (scopingEntityName.Length > 0) { sb.Append("/"); } sb.Append(entityName); if (subType.Length > 0) { sb.AppendFormat(" [{0}]", subType); } entities[entitiesIndex] = sb.ToString(); entitiesIndex++;; } dashboardMetricSeries.SelectedEntities = String.Join(";", entities); dashboardMetricSeries.NumSelectedEntities = entities.Length; } } if (isTokenPropertyNull(dashboardMetricSeriesMetricMatchCriteriaTemplateObject, "metricExpressionTemplate") == false) { JObject dashboardMetricSeriesMetricExpressionTemplateObject = (JObject)dashboardMetricSeriesMetricMatchCriteriaTemplateObject["metricExpressionTemplate"]; dashboardMetricSeries.MetricExpressionType = getStringValueFromJToken(dashboardMetricSeriesMetricExpressionTemplateObject, "metricExpressionType"); dashboardMetricSeries.MetricDisplayName = getStringValueFromJToken(dashboardMetricSeriesMetricExpressionTemplateObject, "displayName"); if (dashboardMetricSeries.MetricDisplayName == "null") { // yes, sometimes that value is saved as string "null" dashboardMetricSeries.MetricDisplayName = String.Empty; } dashboardMetricSeries.FunctionType = getStringValueFromJToken(dashboardMetricSeriesMetricExpressionTemplateObject, "functionType"); dashboardMetricSeries.MetricPath = getStringValueFromJToken(dashboardMetricSeriesMetricExpressionTemplateObject, "relativeMetricPath"); if (dashboardMetricSeries.MetricPath.Length == 0) { // Must be dashboardMetricSeries.MetricExpressionType = Absolute dashboardMetricSeries.MetricPath = getStringValueFromJToken(dashboardMetricSeriesMetricExpressionTemplateObject, "metricPath"); } if (dashboardMetricSeries.NumSelectedEntities == 0) { // Must be dashboardMetricSeries.MetricExpressionType = Absolute if (isTokenPropertyNull(dashboardMetricSeriesMetricExpressionTemplateObject, "scopeEntity") == false) { JObject dashboardMetricSeriesScopeEntityObject = (JObject)dashboardMetricSeriesMetricExpressionTemplateObject["scopeEntity"]; string scopingEntityName = getStringValueFromJToken(dashboardMetricSeriesScopeEntityObject, "scopingEntityName"); string entityName = getStringValueFromJToken(dashboardMetricSeriesScopeEntityObject, "entityName"); string subType = getStringValueFromJToken(dashboardMetricSeriesScopeEntityObject, "subtype"); StringBuilder sb = new StringBuilder(100); sb.Append(scopingEntityName); if (scopingEntityName.Length > 0) { sb.Append("/"); } sb.Append(entityName); if (subType.Length > 0) { sb.AppendFormat(" [{0}]", subType); } dashboardMetricSeries.SelectedEntities = sb.ToString(); dashboardMetricSeries.NumSelectedEntities = 1; } } if (dashboardMetricSeries.MetricExpressionType == "Boolean") { dashboardMetricSeries.ExpressionOperator = getStringValueFromJToken(dashboardMetricSeriesMetricExpressionTemplateObject["operator"], "type"); dashboardMetricSeries.Expression1 = getStringValueOfObjectFromJToken(dashboardMetricSeriesMetricExpressionTemplateObject, "expression1", false); dashboardMetricSeries.Expression2 = getStringValueOfObjectFromJToken(dashboardMetricSeriesMetricExpressionTemplateObject, "expression2", false); } } } dashboardMetricSeriesList.Add(dashboardMetricSeries); } dashboardMetricSeriesAllList.AddRange(dashboardMetricSeriesList); } dashboardWidgetIndex++; } dashboardWidgetsAllList.AddRange(dashboardWidgetsList); dashboard.NumWidgets = dashboardWidgetsList.Count; dashboard.NumAnalyticsWidgets = dashboardWidgetsList.Count(d => d.WidgetType == "AnalyticsWidget"); dashboard.NumEventListWidgets = dashboardWidgetsList.Count(d => d.WidgetType == "EventListWidget"); dashboard.NumGaugeWidgets = dashboardWidgetsList.Count(d => d.WidgetType == "GaugeWidget"); dashboard.NumGraphWidgets = dashboardWidgetsList.Count(d => d.WidgetType == "GraphWidget"); dashboard.NumHealthListWidgets = dashboardWidgetsList.Count(d => d.WidgetType == "HealthListWidget"); dashboard.NumIFrameWidgets = dashboardWidgetsList.Count(d => d.WidgetType == "IFrameWidget"); dashboard.NumImageWidgets = dashboardWidgetsList.Count(d => d.WidgetType == "ImageWidget"); dashboard.NumMetricLabelWidgets = dashboardWidgetsList.Count(d => d.WidgetType == "MetricLabelWidget"); dashboard.NumPieWidgets = dashboardWidgetsList.Count(d => d.WidgetType == "PieWidget"); dashboard.NumTextWidgets = dashboardWidgetsList.Count(d => d.WidgetType == "TextWidget"); } dashboardsList.Add(dashboard); j++; if (j % 100 == 0) { Console.Write("[{0}].", j); } } } loggerConsole.Info("{0} Dashboards", dashboardsList.Count); loggerConsole.Info("{0} Dashboard Widgets", dashboardWidgetsAllList.Count); loggerConsole.Info("{0} Dashboard Widget Time Series", dashboardMetricSeriesAllList.Count); dashboardsList = dashboardsList.OrderBy(d => d.DashboardName).ToList(); FileIOHelper.WriteListToCSVFile(dashboardsList, new DashboardReportMap(), FilePathMap.DashboardsIndexFilePath(jobTarget)); dashboardWidgetsAllList = dashboardWidgetsAllList.OrderBy(d => d.DashboardName).ThenBy(d => d.Index).ToList(); FileIOHelper.WriteListToCSVFile(dashboardWidgetsAllList, new DashboardWidgetReportMap(), FilePathMap.DashboardWidgetsIndexFilePath(jobTarget)); dashboardMetricSeriesAllList = dashboardMetricSeriesAllList.OrderBy(d => d.DashboardName).ThenBy(d => d.Index).ThenBy(d => d.SeriesName).ToList(); FileIOHelper.WriteListToCSVFile(dashboardMetricSeriesAllList, new DashboardMetricSeriesReportMap(), FilePathMap.DashboardMetricSeriesIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + dashboardsList.Count; #endregion #region Combine All for Report CSV // If it is the first one, clear out the combined folder if (reportFolderCleaned == false) { FileIOHelper.DeleteFolder(FilePathMap.ControllerDashboardsReportFolderPath()); Thread.Sleep(1000); FileIOHelper.CreateFolder(FilePathMap.ControllerDashboardsReportFolderPath()); reportFolderCleaned = true; } // Append all the individual report files into one if (File.Exists(FilePathMap.DashboardsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.DashboardsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.DashboardsReportFilePath(), FilePathMap.DashboardsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.DashboardWidgetsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.DashboardWidgetsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.DashboardWidgetsReportFilePath(), FilePathMap.DashboardWidgetsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.DashboardMetricSeriesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.DashboardMetricSeriesIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.DashboardMetricSeriesReportFilePath(), FilePathMap.DashboardMetricSeriesIndexFilePath(jobTarget)); } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } i++; } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(programOptions, jobConfiguration) == false) { return(true); } bool reportFolderCleaned = false; // Process each Controller once int i = 0; var controllers = jobConfiguration.Target.GroupBy(t => t.Controller); foreach (var controllerGroup in controllers) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = controllerGroup.ToList()[0]; StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; stepTimingTarget.NumEntities = 0; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Prepare time range long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From); long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To); int differenceInMinutes = (int)(jobConfiguration.Input.TimeRange.To - jobConfiguration.Input.TimeRange.From).Duration().TotalMinutes; string DEEPLINK_THIS_TIMERANGE = String.Format(DEEPLINK_TIMERANGE_BETWEEN_TIMES, toTimeUnix, fromTimeUnix, differenceInMinutes); #endregion #region Account Summary loggerConsole.Info("Account Summary"); LicenseAccountSummary licenseAccountSummary = new LicenseAccountSummary(); JObject accountSummaryContainerObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.LicenseAccountDataFilePath(jobTarget)); if (accountSummaryContainerObject != null && isTokenPropertyNull(accountSummaryContainerObject, "account") == false) { JObject accountSummaryLicenseObject = (JObject)accountSummaryContainerObject["account"]; licenseAccountSummary.Controller = jobTarget.Controller; licenseAccountSummary.AccountID = getLongValueFromJToken(accountSummaryLicenseObject, "id"); licenseAccountSummary.AccountName = getStringValueFromJToken(accountSummaryLicenseObject, "name"); licenseAccountSummary.AccountNameGlobal = getStringValueFromJToken(accountSummaryLicenseObject, "globalAccountName"); licenseAccountSummary.AccountNameEUM = getStringValueFromJToken(accountSummaryLicenseObject, "eumAccountName"); licenseAccountSummary.AccessKey1 = getStringValueFromJToken(accountSummaryLicenseObject, "accessKey"); licenseAccountSummary.AccessKey2 = getStringValueFromJToken(accountSummaryLicenseObject, "key"); licenseAccountSummary.LicenseKeyEUM = getStringValueFromJToken(accountSummaryLicenseObject, "eumCloudLicenseKey"); licenseAccountSummary.ServiceKeyES = getStringValueFromJToken(accountSummaryLicenseObject, "eumEventServiceKey"); licenseAccountSummary.ExpirationDate = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(accountSummaryLicenseObject, "expirationDate")); licenseAccountSummary.LicenseLink = String.Format(DEEPLINK_LICENSE, licenseAccountSummary.Controller, DEEPLINK_THIS_TIMERANGE); List <LicenseAccountSummary> licenseAccountSummaryList = new List <LicenseAccountSummary>(1); licenseAccountSummaryList.Add(licenseAccountSummary); FileIOHelper.WriteListToCSVFile(licenseAccountSummaryList, new LicenseAccountSummaryReportMap(), FilePathMap.LicenseAccountIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + licenseAccountSummaryList.Count; } #endregion #region Account Level License Entitlements and Usage loggerConsole.Info("Account License Summary and Usage"); List <License> licensesList = new List <License>(24); // Typically there are 12 values per hour, if we extracted every 5 minute things List <LicenseValue> licenseValuesGlobalList = new List <LicenseValue>(jobConfiguration.Input.HourlyTimeRanges.Count * 12 * licensesList.Count); JObject licenseModulesContainer = FileIOHelper.LoadJObjectFromFile(FilePathMap.LicenseModulesDataFilePath(jobTarget)); if (licenseModulesContainer != null && isTokenPropertyNull(licenseModulesContainer, "modules") == false) { JArray licenseModulesArray = (JArray)licenseModulesContainer["modules"]; foreach (JObject licenseModuleObject in licenseModulesArray) { string licenseModuleName = getStringValueFromJToken(licenseModuleObject, "name"); License license = new License(); license.Controller = jobTarget.Controller; license.AccountID = licenseAccountSummary.AccountID; license.AccountName = licenseAccountSummary.AccountName; // Duration license.Duration = (int)(jobConfiguration.Input.TimeRange.To - jobConfiguration.Input.TimeRange.From).Duration().TotalMinutes; license.From = jobConfiguration.Input.TimeRange.From.ToLocalTime(); license.To = jobConfiguration.Input.TimeRange.To.ToLocalTime(); license.FromUtc = jobConfiguration.Input.TimeRange.From; license.ToUtc = jobConfiguration.Input.TimeRange.To; license.AgentType = licenseModuleName; JObject licenseModulePropertiesContainer = FileIOHelper.LoadJObjectFromFile(FilePathMap.LicenseModulePropertiesDataFilePath(jobTarget, licenseModuleName)); if (licenseModulePropertiesContainer != null && isTokenPropertyNull(licenseModulePropertiesContainer, "properties") == false) { foreach (JObject licensePropertyObject in licenseModulePropertiesContainer["properties"]) { string valueOfProperty = getStringValueFromJToken(licensePropertyObject, "value"); switch (getStringValueFromJToken(licensePropertyObject, "name")) { case "expiry-date": long expiryDateLong = 0; if (long.TryParse(valueOfProperty, out expiryDateLong) == true) { license.ExpirationDate = UnixTimeHelper.ConvertFromUnixTimestamp(expiryDateLong); } break; case "licensing-model": license.Model = valueOfProperty; break; case "edition": license.Edition = valueOfProperty; break; case "number-of-provisioned-licenses": long numberOfLicensesLong = 0; if (long.TryParse(valueOfProperty, out numberOfLicensesLong) == true) { license.Provisioned = numberOfLicensesLong; } break; case "maximum-allowed-licenses": long maximumNumberOfLicensesLong = 0; if (long.TryParse(valueOfProperty, out maximumNumberOfLicensesLong) == true) { license.MaximumAllowed = maximumNumberOfLicensesLong; } break; case "data-retention-period": int dataRetentionPeriodInt = 0; if (int.TryParse(valueOfProperty, out dataRetentionPeriodInt) == true) { license.Retention = dataRetentionPeriodInt; } break; default: break; } } } List <LicenseValue> licenseValuesList = new List <LicenseValue>(jobConfiguration.Input.HourlyTimeRanges.Count * 12); JObject licenseModuleUsagesContainer = FileIOHelper.LoadJObjectFromFile(FilePathMap.LicenseModuleUsagesDataFilePath(jobTarget, licenseModuleName)); if (licenseModuleUsagesContainer != null && isTokenPropertyNull(licenseModuleUsagesContainer, "usages") == false) { foreach (JObject licenseUsageObject in licenseModuleUsagesContainer["usages"]) { LicenseValue licenseValue = new LicenseValue(); licenseValue.Controller = license.Controller; licenseValue.AccountName = license.AccountName; licenseValue.AccountID = license.AccountID; licenseValue.AgentType = license.AgentType; licenseValue.RuleName = "Account"; licenseValue.LicenseEventTimeUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(licenseUsageObject, "createdOn")); licenseValue.LicenseEventTime = licenseValue.LicenseEventTimeUtc.ToLocalTime(); licenseValue.Min = getLongValueFromJToken(licenseUsageObject, "minUnitsUsed"); licenseValue.Max = getLongValueFromJToken(licenseUsageObject, "maxUnitsUsed"); licenseValue.Average = getLongValueFromJToken(licenseUsageObject, "avgUnitsUsed"); licenseValue.Total = getLongValueFromJToken(licenseUsageObject, "totalUnitsUsed"); licenseValue.Samples = getLongValueFromJToken(licenseUsageObject, "sampleCount"); licenseValuesList.Add(licenseValue); } } // Do the counts and averages from per hour consumption to the total line if (licenseValuesList.Count > 0) { license.Average = (long)Math.Round((double)((double)licenseValuesList.Sum(mv => mv.Average) / (double)licenseValuesList.Count), 0); license.Min = licenseValuesList.Min(l => l.Min); license.Max = licenseValuesList.Max(l => l.Max); } licensesList.Add(license); licenseValuesGlobalList.AddRange(licenseValuesList); } } stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + licensesList.Count; licensesList = licensesList.OrderBy(l => l.AgentType).ToList(); FileIOHelper.WriteListToCSVFile(licensesList, new LicenseReportMap(), FilePathMap.LicensesIndexFilePath(jobTarget)); FileIOHelper.WriteListToCSVFile(licenseValuesGlobalList, new LicenseValueReportMap(), FilePathMap.LicenseUsageAccountIndexFilePath(jobTarget)); #endregion #region License Rules loggerConsole.Info("License Rules"); #region Preload the application and machine mapping Dictionary <string, string> applicationsUsedByRules = null; Dictionary <string, string> simMachinesUsedByRules = null; JArray licenseApplicationsReferenceArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.LicenseApplicationsDataFilePath(jobTarget)); JArray licenseSIMMachinesReferenceArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.LicenseSIMMachinesDataFilePath(jobTarget)); if (licenseApplicationsReferenceArray == null) { applicationsUsedByRules = new Dictionary <string, string>(0); } else { applicationsUsedByRules = new Dictionary <string, string>(licenseApplicationsReferenceArray.Count); foreach (JObject applicationObject in licenseApplicationsReferenceArray) { applicationsUsedByRules.Add(getStringValueFromJToken(applicationObject["objectReference"], "id"), getStringValueFromJToken(applicationObject, "name")); } } if (licenseSIMMachinesReferenceArray == null) { simMachinesUsedByRules = new Dictionary <string, string>(0); } else { simMachinesUsedByRules = new Dictionary <string, string>(licenseSIMMachinesReferenceArray.Count); foreach (JObject machineObject in licenseSIMMachinesReferenceArray) { simMachinesUsedByRules.Add(getStringValueFromJToken(machineObject["objectReference"], "id"), getStringValueFromJToken(machineObject, "name")); } } List <ControllerApplication> controllerApplicationsList = FileIOHelper.ReadListFromCSVFile <ControllerApplication>(FilePathMap.ControllerApplicationsIndexFilePath(jobTarget), new ControllerApplicationReportMap()); #endregion List <LicenseRule> licenseRulesList = new List <LicenseRule>(10); List <LicenseRuleScope> licenseRuleScopesList = new List <LicenseRuleScope>(100); JArray licenseRulesArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.LicenseRulesDataFilePath(jobTarget)); if (licenseRulesArray != null) { foreach (JObject licenseRuleObject in licenseRulesArray) { string ruleID = getStringValueFromJToken(licenseRuleObject, "id"); string ruleName = getStringValueFromJToken(licenseRuleObject, "name"); JObject licenseRuleDetailsObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.LicenseRuleConfigurationDataFilePath(jobTarget, ruleName, ruleID)); if (licenseRuleDetailsObject != null) { LicenseRule licenseRuleTemplate = new LicenseRule(); licenseRuleTemplate.Controller = jobTarget.Controller; licenseRuleTemplate.AccountID = licenseAccountSummary.AccountID; licenseRuleTemplate.AccountName = licenseAccountSummary.AccountName; // Duration licenseRuleTemplate.Duration = (int)(jobConfiguration.Input.TimeRange.To - jobConfiguration.Input.TimeRange.From).Duration().TotalMinutes; licenseRuleTemplate.From = jobConfiguration.Input.TimeRange.From.ToLocalTime(); licenseRuleTemplate.To = jobConfiguration.Input.TimeRange.To.ToLocalTime(); licenseRuleTemplate.FromUtc = jobConfiguration.Input.TimeRange.From; licenseRuleTemplate.ToUtc = jobConfiguration.Input.TimeRange.To; licenseRuleTemplate.RuleName = getStringValueFromJToken(licenseRuleObject, "name"); licenseRuleTemplate.RuleID = getStringValueFromJToken(licenseRuleObject, "id"); licenseRuleTemplate.AccessKey = getStringValueFromJToken(licenseRuleObject, "access_key"); licenseRuleTemplate.RuleLicenses = getLongValueFromJToken(licenseRuleObject, "total_licenses"); licenseRuleTemplate.RulePeak = getLongValueFromJToken(licenseRuleObject, "peak_usage"); if (isTokenPropertyNull(licenseRuleDetailsObject, "constraints") == false) { JArray licenseRuleConstraintsArray = (JArray)licenseRuleDetailsObject["constraints"]; foreach (JObject licenseConstraintObject in licenseRuleConstraintsArray) { LicenseRuleScope licenseRuleScope = new LicenseRuleScope(); licenseRuleScope.Controller = licenseRuleTemplate.Controller; licenseRuleScope.AccountID = licenseRuleTemplate.AccountID; licenseRuleScope.AccountName = licenseRuleTemplate.AccountName; licenseRuleScope.RuleName = licenseRuleTemplate.RuleName; licenseRuleScope.RuleID = licenseRuleTemplate.RuleID; licenseRuleScope.ScopeSelector = getStringValueFromJToken(licenseConstraintObject, "constraint_type"); string scopeType = getStringValueFromJToken(licenseConstraintObject, "entity_type_id"); if (scopeType == "com.appdynamics.modules.apm.topology.impl.persistenceapi.model.ApplicationEntity") { licenseRuleScope.EntityType = "Application"; } else if (scopeType == "com.appdynamics.modules.apm.topology.impl.persistenceapi.model.MachineEntity") { licenseRuleScope.EntityType = "Machine"; } if (licenseRuleScope.ScopeSelector == "ALLOW_ALL") { licenseRuleScope.MatchType = "EQUALS"; if (scopeType == "com.appdynamics.modules.apm.topology.impl.persistenceapi.model.ApplicationEntity") { licenseRuleScope.EntityName = "[Any Application]"; licenseRuleTemplate.NumApplications++; } else if (scopeType == "com.appdynamics.modules.apm.topology.impl.persistenceapi.model.MachineEntity") { licenseRuleScope.EntityName = "[Any Machine]"; licenseRuleTemplate.NumServers++; } licenseRuleScopesList.Add(licenseRuleScope); } else if (isTokenPropertyNull(licenseConstraintObject, "match_conditions") == false) { JArray licenseRuleMatcheConditionsArray = (JArray)licenseConstraintObject["match_conditions"]; foreach (JObject licenseRuleMatchConditionObject in licenseRuleMatcheConditionsArray) { LicenseRuleScope licenseRuleScopeMatchCondition = licenseRuleScope.Clone(); licenseRuleScopeMatchCondition.MatchType = getStringValueFromJToken(licenseRuleMatchConditionObject, "match_type"); if (licenseRuleScopeMatchCondition.MatchType == "EQUALS" && getStringValueFromJToken(licenseRuleMatchConditionObject, "attribute_type") == "ID") { string applicationID = getStringValueFromJToken(licenseRuleMatchConditionObject, "match_string"); if (scopeType == "com.appdynamics.modules.apm.topology.impl.persistenceapi.model.ApplicationEntity") { if (applicationsUsedByRules.ContainsKey(applicationID) == true && controllerApplicationsList != null) { ControllerApplication controllerApplication = controllerApplicationsList.Where(a => a.ApplicationName == applicationsUsedByRules[applicationID]).FirstOrDefault(); if (controllerApplication != null) { licenseRuleScopeMatchCondition.EntityName = controllerApplication.ApplicationName; licenseRuleScopeMatchCondition.EntityType = controllerApplication.Type; licenseRuleScopeMatchCondition.EntityID = controllerApplication.ApplicationID; } } licenseRuleTemplate.NumApplications++; } else if (scopeType == "com.appdynamics.modules.apm.topology.impl.persistenceapi.model.MachineEntity") { if (simMachinesUsedByRules.ContainsKey(applicationID) == true) { licenseRuleScopeMatchCondition.EntityName = simMachinesUsedByRules[applicationID]; } licenseRuleTemplate.NumServers++; } } else { licenseRuleScopeMatchCondition.EntityName = getStringValueFromJToken(licenseRuleMatchConditionObject, "match_string"); licenseRuleScopeMatchCondition.EntityID = -1; if (scopeType == "com.appdynamics.modules.apm.topology.impl.persistenceapi.model.ApplicationEntity") { licenseRuleTemplate.NumApplications++; } else if (scopeType == "com.appdynamics.modules.apm.topology.impl.persistenceapi.model.MachineEntity") { licenseRuleTemplate.NumServers++; } } licenseRuleScopesList.Add(licenseRuleScopeMatchCondition); } } } } if (isTokenPropertyNull(licenseRuleDetailsObject, "entitlements") == false) { JArray licenseRuleEntitlementsArray = (JArray)licenseRuleDetailsObject["entitlements"]; foreach (JObject licenseEntitlementObject in licenseRuleEntitlementsArray) { LicenseRule licenseRule = licenseRuleTemplate.Clone(); licenseRule.Licenses = getLongValueFromJToken(licenseEntitlementObject, "number_of_licenses"); licenseRule.AgentType = getStringValueFromJToken(licenseEntitlementObject, "license_module_type"); licenseRulesList.Add(licenseRule); } } } } licenseRulesList = licenseRulesList.OrderBy(l => l.RuleName).ThenBy(l => l.AgentType).ToList(); FileIOHelper.WriteListToCSVFile(licenseRulesList, new LicenseRuleReportMap(), FilePathMap.LicenseRulesIndexFilePath(jobTarget)); licenseRuleScopesList = licenseRuleScopesList.OrderBy(l => l.RuleName).ThenBy(l => l.ScopeSelector).ThenBy(l => l.EntityType).ThenBy(l => l.EntityName).ToList(); FileIOHelper.WriteListToCSVFile(licenseRuleScopesList, new LicenseRuleScopeReportMap(), FilePathMap.LicenseRuleScopesIndexFilePath(jobTarget)); } #endregion #region License Rule consumption details loggerConsole.Info("Rules License Consumption"); // Typically there are 12 values per hour // Assume there will be 16 different license types List <LicenseValue> licenseValuesRulesList = new List <LicenseValue>(jobConfiguration.Input.HourlyTimeRanges.Count * 12 * 16 * licenseRulesList.Count); foreach (LicenseRule licenseRule in licenseRulesList) { JObject licenseValuesForRuleContainerObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.LicenseRuleUsageDataFilePath(jobTarget, licenseRule.RuleName, licenseRule.RuleID)); if (licenseValuesForRuleContainerObject != null) { if (isTokenPropertyNull(licenseValuesForRuleContainerObject, "apmStackGraphViewData") == false) { // Parse the APM results first JArray licenseValuesForRuleAPMContainerArray = (JArray)licenseValuesForRuleContainerObject["apmStackGraphViewData"]; if (licenseValuesForRuleAPMContainerArray != null) { foreach (JObject licenseValuesContainerObject in licenseValuesForRuleAPMContainerArray) { string licenseType = getStringValueFromJToken(licenseValuesContainerObject, "licenseModuleType"); // If we have the license looked up, look through the values if (licenseType != String.Empty && isTokenPropertyNull(licenseValuesContainerObject, "graphPoints") == false) { // Looks like [[ 1555484400000, 843 ], [ 1555488000000, 843 ]] JArray licenseValuesAPMArray = (JArray)licenseValuesContainerObject["graphPoints"]; List <LicenseValue> licenseValuesList = parseLicenseValuesArray(licenseValuesAPMArray, licenseRule, licenseType); if (licenseValuesList != null) { licenseValuesRulesList.AddRange(licenseValuesList); } } } } } if (isTokenPropertyNull(licenseValuesForRuleContainerObject, "nonApmModuleDetailViewData") == false) { // Parse the non-APM results second JArray licenseValuesForRuleNonAPMContainerArray = (JArray)licenseValuesForRuleContainerObject["nonApmModuleDetailViewData"]; if (licenseValuesForRuleNonAPMContainerArray != null) { foreach (JObject licenseValuesContainerObject in licenseValuesForRuleNonAPMContainerArray) { string licenseType = getStringValueFromJToken(licenseValuesContainerObject, "licenseModuleType"); // If we have the license looked up, look through the values if (licenseType != String.Empty && isTokenPropertyNull(licenseValuesContainerObject, "graphPoints") == false) { // Looks like [[ 1555484400000, 843 ], [ 1555488000000, 843 ]] JArray licenseValuesAPMArray = (JArray)licenseValuesContainerObject["graphPoints"]; List <LicenseValue> licenseValuesList = parseLicenseValuesArray(licenseValuesAPMArray, licenseRule, licenseType); if (licenseValuesList != null) { licenseValuesRulesList.AddRange(licenseValuesList); } } } } } } } FileIOHelper.WriteListToCSVFile(licenseValuesRulesList, new LicenseRuleValueReportMap(), FilePathMap.LicenseUsageRulesIndexFilePath(jobTarget)); #endregion #region Combine All for Report CSV // If it is the first one, clear out the combined folder if (reportFolderCleaned == false) { FileIOHelper.DeleteFolder(FilePathMap.ControllerLicensesReportFolderPath()); Thread.Sleep(1000); FileIOHelper.CreateFolder(FilePathMap.ControllerLicensesReportFolderPath()); reportFolderCleaned = true; } // Append all the individual report files into one if (File.Exists(FilePathMap.LicenseAccountIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.LicenseAccountIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.LicenseAccountReportFilePath(), FilePathMap.LicenseAccountIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.LicensesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.LicensesIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.LicensesReportFilePath(), FilePathMap.LicensesIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.LicenseRulesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.LicenseRulesIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.LicenseRulesReportFilePath(), FilePathMap.LicenseRulesIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.LicenseUsageAccountIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.LicenseUsageAccountIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.LicenseUsageAccountReportFilePath(), FilePathMap.LicenseUsageAccountIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.LicenseUsageRulesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.LicenseUsageRulesIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.LicenseUsageRulesReportFilePath(), FilePathMap.LicenseUsageRulesIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.LicenseRuleScopesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.LicenseRuleScopesIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.LicenseRuleScopesReportFilePath(), FilePathMap.LicenseRuleScopesIndexFilePath(jobTarget)); } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } i++; } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(jobConfiguration) == false) { return(true); } bool reportFolderCleaned = false; // Process each Controller once int i = 0; var controllers = jobConfiguration.Target.GroupBy(t => t.Controller); foreach (var controllerGroup in controllers) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = controllerGroup.ToList()[0]; StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; stepTimingTarget.NumEntities = 0; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Controller Version loggerConsole.Info("Controller Version"); // Create this row ControllerSummary controllerSummary = new ControllerSummary(); controllerSummary.Controller = jobTarget.Controller; controllerSummary.ControllerLink = String.Format(DEEPLINK_CONTROLLER, controllerSummary.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); // Lookup version // Load the configuration.xml from the child to parse the version XmlDocument configXml = FileIOHelper.LoadXmlDocumentFromFile(FilePathMap.ControllerVersionDataFilePath(jobTarget)); if (configXml != null) { //<serverstatus version="1" vendorid=""> // <available>true</available> // <serverid/> // <serverinfo> // <vendorname>AppDynamics</vendorname> // <productname>AppDynamics Application Performance Management</productname> // <serverversion>004-004-001-000</serverversion> // <implementationVersion>Controller v4.4.1.0 Build 164 Commit 6e1fd94d18dc87c1ecab2da573f98cea49d31c3a</implementationVersion> // </serverinfo> // <startupTimeInSeconds>19</startupTimeInSeconds> //</serverstatus> string controllerVersion = configXml.SelectSingleNode("serverstatus/serverinfo/serverversion").InnerText; string[] controllerVersionArray = controllerVersion.Split('-'); int[] controllerVersionArrayNum = new int[controllerVersionArray.Length]; for (int j = 0; j < controllerVersionArray.Length; j++) { controllerVersionArrayNum[j] = Convert.ToInt32(controllerVersionArray[j]); } controllerVersion = String.Join(".", controllerVersionArrayNum); controllerSummary.Version = controllerVersion; controllerSummary.VersionDetail = configXml.SelectSingleNode("serverstatus/serverinfo/implementationVersion").InnerText; controllerSummary.StartupTime = Convert.ToInt32(configXml.SelectSingleNode("serverstatus/startupTimeInSeconds").InnerText); } else { controllerSummary.Version = "No config data"; } #endregion #region All Applications JObject allApplicationsContainerObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.AllApplicationsDataFilePath(jobTarget)); JArray mobileApplicationsArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.MOBILEApplicationsDataFilePath(jobTarget)); List <ControllerApplication> controllerApplicationsList = new List <ControllerApplication>(100); if (isTokenPropertyNull(allApplicationsContainerObject, "apmApplications") == false) { loggerConsole.Info("Index List of APM Applications"); foreach (JObject applicationObject in allApplicationsContainerObject["apmApplications"]) { ControllerApplication controllerApplication = new ControllerApplication(); controllerApplication.Controller = jobTarget.Controller; populateApplicationInfo(applicationObject, controllerApplication); controllerApplication.Type = APPLICATION_TYPE_APM; controllerApplicationsList.Add(controllerApplication); } } if (isTokenPropertyNull(allApplicationsContainerObject, "eumWebApplications") == false) { loggerConsole.Info("Index List of WEB Applications"); foreach (JObject applicationObject in allApplicationsContainerObject["eumWebApplications"]) { ControllerApplication controllerApplication = new ControllerApplication(); controllerApplication.Controller = jobTarget.Controller; populateApplicationInfo(applicationObject, controllerApplication); controllerApplication.Type = APPLICATION_TYPE_WEB; controllerApplicationsList.Add(controllerApplication); } } if (isTokenPropertyNull(allApplicationsContainerObject, "iotApplications") == false) { loggerConsole.Info("Index List of IOT Applications"); foreach (JObject applicationObject in allApplicationsContainerObject["iotApplications"]) { ControllerApplication controllerApplication = new ControllerApplication(); controllerApplication.Controller = jobTarget.Controller; populateApplicationInfo(applicationObject, controllerApplication); controllerApplication.Type = APPLICATION_TYPE_IOT; controllerApplicationsList.Add(controllerApplication); } } if (isTokenPropertyNull(allApplicationsContainerObject, "mobileAppContainers") == false) { loggerConsole.Info("Index List of MOBILE Applications"); foreach (JObject applicationObject in allApplicationsContainerObject["mobileAppContainers"]) { ControllerApplication controllerApplication = new ControllerApplication(); controllerApplication.Controller = jobTarget.Controller; populateApplicationInfo(applicationObject, controllerApplication); controllerApplication.Type = APPLICATION_TYPE_MOBILE; if (controllerApplicationsList.Where(a => a.ApplicationID == controllerApplication.ApplicationID).Count() == 0) { controllerApplicationsList.Add(controllerApplication); } // Now go through children if (mobileApplicationsArray != null) { JToken mobileApplicationContainerObject = mobileApplicationsArray.Where(a => getLongValueFromJToken(a, "applicationId") == controllerApplication.ApplicationID).FirstOrDefault(); if (mobileApplicationContainerObject != null) { foreach (JObject mobileApplicationChildJSON in mobileApplicationContainerObject["children"]) { ControllerApplication controllerApplicationChild = controllerApplication.Clone(); controllerApplicationChild.ParentApplicationID = controllerApplicationChild.ApplicationID; controllerApplicationChild.ApplicationName = getStringValueFromJToken(mobileApplicationChildJSON, "name"); controllerApplicationChild.ApplicationID = getLongValueFromJToken(mobileApplicationChildJSON, "mobileAppId"); controllerApplicationsList.Add(controllerApplicationChild); } } } } } if (isTokenPropertyNull(allApplicationsContainerObject, "dbMonApplication") == false) { loggerConsole.Info("Index DB Application"); JObject applicationObject = (JObject)allApplicationsContainerObject["dbMonApplication"]; ControllerApplication controllerApplication = new ControllerApplication(); controllerApplication.Controller = jobTarget.Controller; populateApplicationInfo(applicationObject, controllerApplication); controllerApplication.Type = APPLICATION_TYPE_DB; controllerApplicationsList.Add(controllerApplication); } if (isTokenPropertyNull(allApplicationsContainerObject, "simApplication") == false) { loggerConsole.Info("Index SIM Application"); JObject applicationObject = (JObject)allApplicationsContainerObject["simApplication"]; ControllerApplication controllerApplication = new ControllerApplication(); controllerApplication.Controller = jobTarget.Controller; populateApplicationInfo(applicationObject, controllerApplication); controllerApplication.Type = APPLICATION_TYPE_SIM; controllerApplicationsList.Add(controllerApplication); } if (isTokenPropertyNull(allApplicationsContainerObject, "analyticsApplication") == false) { loggerConsole.Info("Index BIQ Application"); JObject applicationObject = (JObject)allApplicationsContainerObject["analyticsApplication"]; ControllerApplication controllerApplication = new ControllerApplication(); controllerApplication.Controller = jobTarget.Controller; populateApplicationInfo(applicationObject, controllerApplication); controllerApplication.Type = APPLICATION_TYPE_BIQ; controllerApplicationsList.Add(controllerApplication); } // Sort them controllerApplicationsList = controllerApplicationsList.OrderBy(o => o.Type).ThenBy(o => o.ApplicationName).ToList(); FileIOHelper.WriteListToCSVFile(controllerApplicationsList, new ControllerApplicationReportMap(), FilePathMap.ControllerApplicationsIndexFilePath(jobTarget)); controllerSummary.NumApps = controllerApplicationsList.Count; controllerSummary.NumAPMApps = controllerApplicationsList.Where(a => a.Type == APPLICATION_TYPE_APM).Count(); controllerSummary.NumWEBApps = controllerApplicationsList.Where(a => a.Type == APPLICATION_TYPE_WEB).Count(); controllerSummary.NumMOBILEApps = controllerApplicationsList.Where(a => a.Type == APPLICATION_TYPE_MOBILE).Count(); controllerSummary.NumSIMApps = controllerApplicationsList.Where(a => a.Type == APPLICATION_TYPE_SIM).Count(); controllerSummary.NumDBApps = controllerApplicationsList.Where(a => a.Type == APPLICATION_TYPE_DB).Count(); controllerSummary.NumBIQApps = controllerApplicationsList.Where(a => a.Type == APPLICATION_TYPE_BIQ).Count(); controllerSummary.NumIOTApps = controllerApplicationsList.Where(a => a.Type == APPLICATION_TYPE_IOT).Count(); List <ControllerSummary> controllerList = new List <ControllerSummary>(1); controllerList.Add(controllerSummary); FileIOHelper.WriteListToCSVFile(controllerList, new ControllerSummaryReportMap(), FilePathMap.ControllerSummaryIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + controllerApplicationsList.Count; #endregion #region Combine All for Report CSV // If it is the first one, clear out the combined folder if (reportFolderCleaned == false) { FileIOHelper.DeleteFolder(FilePathMap.ControllerEntitiesReportFolderPath()); Thread.Sleep(1000); FileIOHelper.CreateFolder(FilePathMap.ControllerEntitiesReportFolderPath()); reportFolderCleaned = true; } // Append all the individual report files into one if (File.Exists(FilePathMap.ControllerSummaryIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.ControllerSummaryIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.ControllerSummaryReportFilePath(), FilePathMap.ControllerSummaryIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.ControllerApplicationsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.ControllerApplicationsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.ControllerApplicationsReportFilePath(), FilePathMap.ControllerApplicationsIndexFilePath(jobTarget)); } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } i++; } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(jobConfiguration) == false) { return(true); } if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_DB) == 0) { return(true); } bool reportFolderCleaned = false; // Process each Controller once int i = 0; var controllers = jobConfiguration.Target.Where(t => t.Type == APPLICATION_TYPE_DB).ToList().GroupBy(t => t.Controller); foreach (var controllerGroup in controllers) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = controllerGroup.ToList()[0]; if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_DB) { continue; } StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Database Collector Definitions List <DBCollectorDefinition> dbCollectorDefinitionsList = null; JArray dbCollectorDefinitionsArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.DBCollectorDefinitionsDataFilePath(jobTarget)); if (dbCollectorDefinitionsArray != null) { loggerConsole.Info("Index List of DB Collector Definitions ({0} entities)", dbCollectorDefinitionsArray.Count); dbCollectorDefinitionsList = new List <DBCollectorDefinition>(dbCollectorDefinitionsArray.Count); foreach (JToken dbCollectorDefinitionToken in dbCollectorDefinitionsArray) { if (isTokenPropertyNull(dbCollectorDefinitionToken, "config") == false) { JToken dbCollectorDefinitionConfigToken = dbCollectorDefinitionToken["config"]; DBCollectorDefinition dbCollectorDefinition = new DBCollectorDefinition(); dbCollectorDefinition.Controller = jobTarget.Controller; dbCollectorDefinition.CollectorName = getStringValueFromJToken(dbCollectorDefinitionConfigToken, "name"); dbCollectorDefinition.CollectorType = getStringValueFromJToken(dbCollectorDefinitionConfigToken, "type"); dbCollectorDefinition.CollectorStatus = getStringValueFromJToken(dbCollectorDefinitionToken, "collectorStatus"); dbCollectorDefinition.AgentName = getStringValueFromJToken(dbCollectorDefinitionConfigToken, "agentName"); dbCollectorDefinition.Host = getStringValueFromJToken(dbCollectorDefinitionConfigToken, "hostname"); dbCollectorDefinition.Port = getIntValueFromJToken(dbCollectorDefinitionConfigToken, "port"); dbCollectorDefinition.UserName = getStringValueFromJToken(dbCollectorDefinitionConfigToken, "username"); dbCollectorDefinition.IsEnabled = getBoolValueFromJToken(dbCollectorDefinitionConfigToken, "enabled"); dbCollectorDefinition.IsLoggingEnabled = getBoolValueFromJToken(dbCollectorDefinitionConfigToken, "loggingEnabled"); dbCollectorDefinition.DatabaseName = getStringValueFromJToken(dbCollectorDefinitionConfigToken, "databaseName"); dbCollectorDefinition.FailoverPartner = getStringValueFromJToken(dbCollectorDefinitionConfigToken, "failoverPartner"); dbCollectorDefinition.SID = getStringValueFromJToken(dbCollectorDefinitionConfigToken, "sid"); dbCollectorDefinition.CustomConnectionString = getStringValueFromJToken(dbCollectorDefinitionConfigToken, "customConnectionString"); dbCollectorDefinition.UseWindowsAuth = getBoolValueFromJToken(dbCollectorDefinitionConfigToken, "useWindowsAuth"); dbCollectorDefinition.ConnectAsSysDBA = getBoolValueFromJToken(dbCollectorDefinitionConfigToken, "connectAsSysdba"); dbCollectorDefinition.UseServiceName = getBoolValueFromJToken(dbCollectorDefinitionConfigToken, "useServiceName"); dbCollectorDefinition.UseSSL = getBoolValueFromJToken(dbCollectorDefinitionConfigToken, "useSSL"); dbCollectorDefinition.IsEnterpriseDB = getBoolValueFromJToken(dbCollectorDefinitionConfigToken, "enterpriseDB"); dbCollectorDefinition.IsOSMonitoringEnabled = getBoolValueFromJToken(dbCollectorDefinitionConfigToken, "enableOSMonitor"); dbCollectorDefinition.UseLocalWMI = getBoolValueFromJToken(dbCollectorDefinitionConfigToken, "useLocalWMI"); dbCollectorDefinition.HostOS = getStringValueFromJToken(dbCollectorDefinitionConfigToken, "hostOS"); dbCollectorDefinition.HostDomain = getStringValueFromJToken(dbCollectorDefinitionConfigToken, "hostDomain"); dbCollectorDefinition.HostUserName = getStringValueFromJToken(dbCollectorDefinitionConfigToken, "hostUsername"); dbCollectorDefinition.UseCertificateAuth = getBoolValueFromJToken(dbCollectorDefinitionConfigToken, "certificateAuth"); dbCollectorDefinition.SSHPort = getIntValueFromJToken(dbCollectorDefinitionConfigToken, "sshPort"); dbCollectorDefinition.DBInstanceID = getStringValueFromJToken(dbCollectorDefinitionConfigToken, "dbInstanceIdentifier"); dbCollectorDefinition.Region = getStringValueFromJToken(dbCollectorDefinitionConfigToken, "region"); dbCollectorDefinition.RemoveLiterals = getBoolValueFromJToken(dbCollectorDefinitionConfigToken, "removeLiterals"); dbCollectorDefinition.IsLDAPEnabled = getBoolValueFromJToken(dbCollectorDefinitionConfigToken, "ldapEnabled"); dbCollectorDefinition.CreatedBy = getStringValueFromJToken(dbCollectorDefinitionConfigToken, "createdBy"); dbCollectorDefinition.CreatedOn = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(dbCollectorDefinitionConfigToken, "createdOn")); dbCollectorDefinition.ModifiedBy = getStringValueFromJToken(dbCollectorDefinitionConfigToken, "modifiedBy"); dbCollectorDefinition.ModifiedOn = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(dbCollectorDefinitionConfigToken, "modifiedOn")); dbCollectorDefinition.ConfigID = getLongValueFromJToken(dbCollectorDefinitionToken, "configId"); dbCollectorDefinition.ControllerLink = String.Format(DEEPLINK_CONTROLLER, dbCollectorDefinition.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); dbCollectorDefinition.ApplicationLink = String.Format(DEEPLINK_DB_APPLICATION, dbCollectorDefinition.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); dbCollectorDefinitionsList.Add(dbCollectorDefinition); } } // Sort them dbCollectorDefinitionsList = dbCollectorDefinitionsList.OrderBy(o => o.CollectorType).ThenBy(o => o.CollectorName).ToList(); FileIOHelper.WriteListToCSVFile(dbCollectorDefinitionsList, new DBCollectorDefinitionReportMap(), FilePathMap.DBCollectorDefinitionsIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + dbCollectorDefinitionsList.Count; } #endregion #region Custom Metrics List <DBCustomMetric> dbCustomMetricsList = null; JArray dbCustomMetricsArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.DBCustomMetricsDataFilePath(jobTarget)); if (dbCustomMetricsArray != null) { loggerConsole.Info("Index List of DB Custom Metrics ({0} entities)", dbCustomMetricsArray.Count); dbCustomMetricsList = new List <DBCustomMetric>(dbCustomMetricsArray.Count); foreach (JObject dbCustomMetricObject in dbCustomMetricsArray) { DBCustomMetric dbCustomMetric = new DBCustomMetric(); dbCustomMetric.Controller = jobTarget.Controller; dbCustomMetric.MetricName = getStringValueFromJToken(dbCustomMetricObject, "name"); dbCustomMetric.MetricID = getLongValueFromJToken(dbCustomMetricObject, "id"); dbCustomMetric.Frequency = getIntValueFromJToken(dbCustomMetricObject, "timeIntervalInMin"); dbCustomMetric.IsEvent = getBoolValueFromJToken(dbCustomMetricObject, "isEvent"); dbCustomMetric.Query = getStringValueFromJToken(dbCustomMetricObject, "queryText"); // Get SQL statement type dbCustomMetric.SQLClauseType = getSQLClauseType(dbCustomMetric.Query, 100); // Check other clauses dbCustomMetric.SQLWhere = doesSQLStatementContain(dbCustomMetric.Query, @"\bWHERE\s"); dbCustomMetric.SQLGroupBy = doesSQLStatementContain(dbCustomMetric.Query, @"\bGROUP BY\s"); dbCustomMetric.SQLOrderBy = doesSQLStatementContain(dbCustomMetric.Query, @"\bORDER BY\s"); dbCustomMetric.SQLHaving = doesSQLStatementContain(dbCustomMetric.Query, @"\bHAVING\s"); dbCustomMetric.SQLUnion = doesSQLStatementContain(dbCustomMetric.Query, @"\bUNION\s"); // Get join type if present dbCustomMetric.SQLJoinType = getSQLJoinType(dbCustomMetric.Query); // Now lookup the collector assigned if (dbCollectorDefinitionsList != null) { if (isTokenPropertyNull(dbCustomMetricObject, "dbaemc") == false) { if (isTokenPropertyNull(dbCustomMetricObject["dbaemc"], "dbServerIds") == false) { JValue configIDValue = null; try { configIDValue = (JValue)dbCustomMetricObject["dbaemc"]["dbServerIds"].First(); } catch { } if (configIDValue != null) { long configID = (long)configIDValue; DBCollectorDefinition dbCollectorDefinition = dbCollectorDefinitionsList.Where(d => d.ConfigID == configID).FirstOrDefault(); if (dbCollectorDefinition != null) { dbCustomMetric.CollectorName = dbCollectorDefinition.CollectorName; dbCustomMetric.CollectorType = dbCollectorDefinition.CollectorType; dbCustomMetric.ConfigID = dbCollectorDefinition.ConfigID; } } } } } dbCustomMetricsList.Add(dbCustomMetric); } // Sort them dbCustomMetricsList = dbCustomMetricsList.OrderBy(o => o.CollectorType).ThenBy(o => o.CollectorName).ToList(); FileIOHelper.WriteListToCSVFile(dbCustomMetricsList, new DBCustomMetricReportMap(), FilePathMap.DBCustomMetricsIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + dbCustomMetricsList.Count; } #endregion #region Application loggerConsole.Info("Index Application"); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + 1; DBApplicationConfiguration applicationConfiguration = new DBApplicationConfiguration(); applicationConfiguration.Controller = jobTarget.Controller; applicationConfiguration.ControllerLink = String.Format(DEEPLINK_CONTROLLER, applicationConfiguration.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); applicationConfiguration.ApplicationName = jobTarget.Application; applicationConfiguration.ApplicationLink = String.Format(DEEPLINK_DB_APPLICATION, applicationConfiguration.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); applicationConfiguration.ApplicationID = jobTarget.ApplicationID; if (dbCollectorDefinitionsList != null) { applicationConfiguration.NumCollectorDefinitions = dbCollectorDefinitionsList.Count; } if (dbCustomMetricsList != null) { applicationConfiguration.NumCustomMetrics = dbCustomMetricsList.Count; } List <DBApplicationConfiguration> applicationConfigurationsList = new List <DBApplicationConfiguration>(1); applicationConfigurationsList.Add(applicationConfiguration); FileIOHelper.WriteListToCSVFile(applicationConfigurationsList, new DBApplicationConfigurationReportMap(), FilePathMap.DBApplicationConfigurationIndexFilePath(jobTarget)); #endregion #region Combine All for Report CSV // If it is the first one, clear out the combined folder if (reportFolderCleaned == false) { FileIOHelper.DeleteFolder(FilePathMap.DBConfigurationReportFolderPath()); Thread.Sleep(1000); FileIOHelper.CreateFolder(FilePathMap.DBConfigurationReportFolderPath()); reportFolderCleaned = true; } // Append all the individual report files into one if (File.Exists(FilePathMap.DBCollectorDefinitionsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.DBCollectorDefinitionsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.DBCollectorDefinitionsReportFilePath(), FilePathMap.DBCollectorDefinitionsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.DBCustomMetricsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.DBCustomMetricsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.DBCustomMetricsReportFilePath(), FilePathMap.DBCustomMetricsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.DBApplicationConfigurationIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.DBApplicationConfigurationIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.DBApplicationConfigurationReportFilePath(), FilePathMap.DBApplicationConfigurationIndexFilePath(jobTarget)); } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } i++; } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(jobConfiguration) == false) { return(true); } if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_MOBILE) == 0) { return(true); } bool reportFolderCleaned = false; // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_MOBILE) { continue; } StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Prepare time range long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From); long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To); int differenceInMinutes = (int)(jobConfiguration.Input.TimeRange.To - jobConfiguration.Input.TimeRange.From).Duration().TotalMinutes; string DEEPLINK_THIS_TIMERANGE = String.Format(DEEPLINK_TIMERANGE_BETWEEN_TIMES, toTimeUnix, fromTimeUnix, differenceInMinutes); #endregion #region Network Requests List <MOBILENetworkRequest> networkRequestsList = null; List <MOBILENetworkRequestToBusinessTransaction> networkRequestToBTsList = null; JObject networkRequestsContainerObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.MOBILENetworkRequestsDataFilePath(jobTarget)); if (isTokenPropertyNull(networkRequestsContainerObject, "data") == false) { JArray networkRequestsArray = (JArray)networkRequestsContainerObject["data"]; loggerConsole.Info("Index List of Network Requests ({0} entities)", networkRequestsArray.Count); networkRequestsList = new List <MOBILENetworkRequest>(networkRequestsArray.Count); networkRequestToBTsList = new List <MOBILENetworkRequestToBusinessTransaction>(networkRequestsArray.Count * 4); foreach (JObject networkRequestObject in networkRequestsArray) { MOBILENetworkRequest networkRequest = new MOBILENetworkRequest(); networkRequest.Controller = jobTarget.Controller; networkRequest.ControllerLink = String.Format(DEEPLINK_CONTROLLER, jobTarget.Controller, DEEPLINK_THIS_TIMERANGE); networkRequest.ApplicationName = jobTarget.Application; networkRequest.ApplicationID = jobTarget.ApplicationID; networkRequest.ApplicationLink = String.Format(DEEPLINK_MOBILE_APPLICATION, networkRequest.Controller, jobTarget.ParentApplicationID, networkRequest.ApplicationID, DEEPLINK_THIS_TIMERANGE); networkRequest.RequestName = getStringValueFromJToken(networkRequestObject, "name"); networkRequest.RequestNameInternal = getStringValueFromJToken(networkRequestObject, "internalName"); networkRequest.RequestID = getLongValueFromJToken(networkRequestObject, "addId"); networkRequest.RequestLink = String.Format(DEEPLINK_NETWORK_REQUEST, networkRequest.Controller, jobTarget.ParentApplicationID, networkRequest.ApplicationID, networkRequest.RequestID, DEEPLINK_THIS_TIMERANGE); networkRequest.Platform = getStringValueFromJToken(networkRequestObject, "mobilePlatform"); networkRequest.IsCorrelated = getBoolValueFromJToken(networkRequestObject, "hasServerAgentCorrelation"); networkRequest.Duration = differenceInMinutes; networkRequest.From = jobConfiguration.Input.TimeRange.From.ToLocalTime(); networkRequest.To = jobConfiguration.Input.TimeRange.To.ToLocalTime(); networkRequest.FromUtc = jobConfiguration.Input.TimeRange.From; networkRequest.ToUtc = jobConfiguration.Input.TimeRange.To; networkRequest.IsExcluded = getBoolValueFromJToken(networkRequestObject, "excluded"); // Now to metrics networkRequest.MetricsIDs = new List <long>(16); JObject networkRequestDetailsObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.MOBILENetworkRequestPerformanceDataFilePath(jobTarget, networkRequest.RequestName, networkRequest.RequestID, jobConfiguration.Input.TimeRange)); if (networkRequestDetailsObject != null) { networkRequest.UserExperience = getStringValueFromJToken(networkRequestDetailsObject, "performanceState"); if (isTokenPropertyNull(networkRequestDetailsObject, "networkRequestTime") == false) { networkRequest.ART = getLongValueFromJToken(networkRequestDetailsObject["networkRequestTime"], "value"); if (networkRequest.ART < 0) { networkRequest.ART = 0; } else { networkRequest.MetricsIDs.Add(getLongValueFromJToken(networkRequestDetailsObject["networkRequestTime"], "metricId")); } networkRequest.ARTRange = getDurationRangeAsString(networkRequest.ART); } if (isTokenPropertyNull(networkRequestDetailsObject, "networkRequestTime") == false && isTokenPropertyNull(networkRequestDetailsObject["networkRequestTime"], "graphData") == false && isTokenPropertyNull(networkRequestDetailsObject["networkRequestTime"]["graphData"], "data") == false) { networkRequest.TimeTotal = sumLongValuesInArray((JArray)networkRequestDetailsObject["networkRequestTime"]["graphData"]["data"]); } if (isTokenPropertyNull(networkRequestDetailsObject, "requestsPerMin") == false) { networkRequest.CPM = getLongValueFromJToken(networkRequestDetailsObject["requestsPerMin"], "value"); if (networkRequest.CPM < 0) { networkRequest.CPM = 0; } else { networkRequest.MetricsIDs.Add(getLongValueFromJToken(networkRequestDetailsObject["requestsPerMin"], "metricId")); } } if (isTokenPropertyNull(networkRequestDetailsObject, "totalNumOfRequests") == false) { networkRequest.Calls = getLongValueFromJToken(networkRequestDetailsObject["totalNumOfRequests"], "value"); if (networkRequest.Calls < 0) { networkRequest.Calls = 0; } else { networkRequest.MetricsIDs.Add(getLongValueFromJToken(networkRequestDetailsObject["totalNumOfRequests"], "metricId")); } } if (isTokenPropertyNull(networkRequestDetailsObject, "totalServerTime") == false) { networkRequest.Server = getLongValueFromJToken(networkRequestDetailsObject["totalServerTime"], "value"); if (networkRequest.Server < 0) { networkRequest.Server = 0; } else { networkRequest.MetricsIDs.Add(getLongValueFromJToken(networkRequestDetailsObject["totalServerTime"], "metricId")); } } if (isTokenPropertyNull(networkRequestDetailsObject, "totalNumHttpErrors") == false) { networkRequest.HttpErrors = getLongValueFromJToken(networkRequestDetailsObject["totalNumHttpErrors"], "value"); if (networkRequest.HttpErrors < 0) { networkRequest.HttpErrors = 0; } else { networkRequest.MetricsIDs.Add(getLongValueFromJToken(networkRequestDetailsObject["totalNumHttpErrors"], "metricId")); } } if (isTokenPropertyNull(networkRequestDetailsObject, "httpErrorsPerMin") == false) { networkRequest.HttpEPM = getLongValueFromJToken(networkRequestDetailsObject["httpErrorsPerMin"], "value"); if (networkRequest.HttpEPM < 0) { networkRequest.HttpEPM = 0; } else { networkRequest.MetricsIDs.Add(getLongValueFromJToken(networkRequestDetailsObject["httpErrorsPerMin"], "metricId")); } } if (isTokenPropertyNull(networkRequestDetailsObject, "totalNumNetworkErrors") == false) { networkRequest.NetworkErrors = getLongValueFromJToken(networkRequestDetailsObject["totalNumNetworkErrors"], "value"); if (networkRequest.NetworkErrors < 0) { networkRequest.NetworkErrors = 0; } else { networkRequest.MetricsIDs.Add(getLongValueFromJToken(networkRequestDetailsObject["totalNumNetworkErrors"], "metricId")); } } if (isTokenPropertyNull(networkRequestDetailsObject, "networkErrorsPerMin") == false) { networkRequest.NetworkEPM = getLongValueFromJToken(networkRequestDetailsObject["networkErrorsPerMin"], "value"); if (networkRequest.NetworkEPM < 0) { networkRequest.NetworkEPM = 0; } else { networkRequest.MetricsIDs.Add(getLongValueFromJToken(networkRequestDetailsObject["networkErrorsPerMin"], "metricId")); } } } // Has Activity if (networkRequest.ART == 0 && networkRequest.TimeTotal == 0 && networkRequest.CPM == 0 && networkRequest.Calls == 0) { networkRequest.HasActivity = false; } else { networkRequest.HasActivity = true; } // Create metric link if (networkRequest.MetricsIDs != null && networkRequest.MetricsIDs.Count > 0) { StringBuilder sb = new StringBuilder(256); foreach (long metricID in networkRequest.MetricsIDs) { if (metricID > 0) { sb.Append(String.Format(DEEPLINK_METRIC_APPLICATION_TARGET_METRIC_ID, jobTarget.ParentApplicationID, metricID)); sb.Append(","); } } sb.Remove(sb.Length - 1, 1); networkRequest.MetricLink = String.Format(DEEPLINK_METRIC, networkRequest.Controller, jobTarget.ParentApplicationID, sb.ToString(), DEEPLINK_THIS_TIMERANGE); } // Load Business Transactions if (networkRequestDetailsObject != null) { if (isTokenPropertyNull(networkRequestDetailsObject, "eumPageRelatedBusinessTransactionData") == false) { foreach (JToken btContainerToken in networkRequestDetailsObject["eumPageRelatedBusinessTransactionData"]) { JObject btContainerObject = (JObject)btContainerToken.First(); MOBILENetworkRequestToBusinessTransaction networkRequestToBT = new MOBILENetworkRequestToBusinessTransaction(); networkRequestToBT.Controller = networkRequest.Controller; networkRequestToBT.ControllerLink = networkRequest.ControllerLink; networkRequestToBT.ApplicationName = networkRequest.ApplicationName; networkRequestToBT.ApplicationID = networkRequest.ApplicationID; networkRequestToBT.ApplicationLink = networkRequest.ApplicationLink; networkRequestToBT.RequestName = networkRequest.RequestName; networkRequestToBT.RequestNameInternal = networkRequest.RequestNameInternal; networkRequestToBT.RequestID = networkRequest.RequestID; networkRequestToBT.Duration = networkRequest.Duration; networkRequestToBT.From = networkRequest.From; networkRequestToBT.To = networkRequest.To; networkRequestToBT.FromUtc = networkRequest.FromUtc; networkRequestToBT.ToUtc = networkRequest.ToUtc; networkRequestToBT.BTID = getLongValueFromJToken(btContainerObject, "businessTransactionId"); networkRequestToBT.BTName = getStringValueFromJToken(btContainerObject, "businessTransactionName"); JobTarget jobTargetofAPM = jobConfiguration.Target.Where(j => j.ApplicationID == getLongValueFromJToken(btContainerObject, "applicationId") && j.Type == APPLICATION_TYPE_WEB).FirstOrDefault(); List <APMBusinessTransaction> businessTransactionsList = null; if (jobTargetofAPM != null) { businessTransactionsList = FileIOHelper.ReadListFromCSVFile <APMBusinessTransaction>(FilePathMap.APMBusinessTransactionsIndexFilePath(jobTargetofAPM), new APMBusinessTransactionReportMap()); } if (businessTransactionsList != null && networkRequestToBT.BTID != 0) { APMBusinessTransaction businessTransaction = businessTransactionsList.Where(b => b.BTID == networkRequestToBT.BTID).FirstOrDefault(); if (businessTransaction != null) { networkRequestToBT.BTType = businessTransaction.BTType; networkRequestToBT.TierName = businessTransaction.TierName; networkRequestToBT.TierID = businessTransaction.TierID; } } if (isTokenPropertyNull(btContainerObject, "averageResponseTime") == false) { networkRequestToBT.ART = getLongValueFromJToken(btContainerObject["averageResponseTime"], "value"); if (networkRequestToBT.ART < 0) { networkRequestToBT.ART = 0; } networkRequestToBT.ARTRange = getDurationRangeAsString(networkRequestToBT.ART); } if (isTokenPropertyNull(btContainerObject, "callsPerMinute") == false) { networkRequestToBT.CPM = getLongValueFromJToken(btContainerObject["callsPerMinute"], "value"); if (networkRequestToBT.CPM < 0) { networkRequestToBT.CPM = 0; } } if (isTokenPropertyNull(btContainerObject, "totalNumberOfCalls") == false) { networkRequestToBT.Calls = getLongValueFromJToken(btContainerObject["totalNumberOfCalls"], "value"); if (networkRequestToBT.Calls < 0) { networkRequestToBT.Calls = 0; } } // Has Activity if (networkRequestToBT.ART == 0 && networkRequestToBT.CPM == 0 && networkRequestToBT.Calls == 0) { networkRequestToBT.HasActivity = false; } else { networkRequestToBT.HasActivity = true; } networkRequest.NumBTs++; networkRequestToBTsList.Add(networkRequestToBT); } } } networkRequestsList.Add(networkRequest); } // Sort them networkRequestsList = networkRequestsList.OrderBy(o => o.RequestName).ToList(); FileIOHelper.WriteListToCSVFile(networkRequestsList, new MOBILENetworkRequestReportMap(), FilePathMap.MOBILENetworkRequestsIndexFilePath(jobTarget)); networkRequestToBTsList = networkRequestToBTsList.OrderBy(o => o.RequestName).ThenBy(o => o.TierName).ThenBy(o => o.BTType).ThenBy(o => o.BTName).ToList(); FileIOHelper.WriteListToCSVFile(networkRequestToBTsList, new MOBILENetworkRequestToBusinessTransactionReportMap(), FilePathMap.MOBILENetworkRequestsBusinessTransactionsIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + networkRequestsList.Count; } #endregion #region Application loggerConsole.Info("Index Application"); MOBILEApplication application = new MOBILEApplication(); application.Controller = jobTarget.Controller; application.ControllerLink = String.Format(DEEPLINK_CONTROLLER, jobTarget.Controller, DEEPLINK_THIS_TIMERANGE); application.ApplicationName = jobTarget.Application; application.ApplicationID = jobTarget.ApplicationID; application.ApplicationLink = String.Format(DEEPLINK_MOBILE_APPLICATION, application.Controller, jobTarget.ParentApplicationID, application.ApplicationID, DEEPLINK_THIS_TIMERANGE); if (networkRequestsList != null) { application.NumNetworkRequests = networkRequestsList.Count; application.NumActivity = networkRequestsList.Count(p => p.HasActivity == true); application.NumNoActivity = networkRequestsList.Count(p => p.HasActivity == false); } List <MOBILEApplication> applicationsList = new List <MOBILEApplication>(1); applicationsList.Add(application); FileIOHelper.WriteListToCSVFile(applicationsList, new MOBILEApplicationReportMap(), FilePathMap.MOBILEApplicationsIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + 1; #endregion #region Combine All for Report CSV // If it is the first one, clear out the combined folder if (reportFolderCleaned == false) { FileIOHelper.DeleteFolder(FilePathMap.MOBILEEntitiesReportFolderPath()); Thread.Sleep(1000); FileIOHelper.CreateFolder(FilePathMap.MOBILEEntitiesReportFolderPath()); reportFolderCleaned = true; } // Append all the individual report files into one if (File.Exists(FilePathMap.MOBILEApplicationsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.MOBILEApplicationsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.MOBILEApplicationsReportFilePath(), FilePathMap.MOBILEApplicationsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.MOBILENetworkRequestsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.MOBILENetworkRequestsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.MOBILENetworkRequestsReportFilePath(), FilePathMap.MOBILENetworkRequestsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.MOBILENetworkRequestsBusinessTransactionsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.MOBILENetworkRequestsBusinessTransactionsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.MOBILENetworkRequestsBusinessTransactionsReportFilePath(), FilePathMap.MOBILENetworkRequestsBusinessTransactionsIndexFilePath(jobTarget)); } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(programOptions, jobConfiguration) == false) { return(true); } if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_DB) == 0) { logger.Warn("No {0} targets to process", APPLICATION_TYPE_DB); loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_DB); return(true); } bool reportFolderCleaned = false; List <MetricExtractMapping> entityMetricExtractMappingList = getMetricsExtractMappingList(jobConfiguration); // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_DB) { continue; } StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); int numEntitiesTotal = 0; ParallelOptions parallelOptions = new ParallelOptions(); if (programOptions.ProcessSequentially == true) { parallelOptions.MaxDegreeOfParallelism = 1; } Parallel.Invoke(parallelOptions, () => { #region Database metrics List <DBCollector> dbCollectorsList = FileIOHelper.ReadListFromCSVFile <DBCollector>(FilePathMap.DBCollectorsIndexFilePath(jobTarget), new DBCollectorReportMap()); if (dbCollectorsList != null) { Dictionary <string, DBEntityBase> entitiesDictionaryByName = dbCollectorsList.ToDictionary(e => e.CollectorName, e => (DBEntityBase)e); Dictionary <string, List <MetricValue> > metricValuesDictionary = new Dictionary <string, List <MetricValue> >(); for (int j = 0; j < jobConfiguration.Input.HourlyTimeRanges.Count; j++) { JobTimeRange jobTimeRange = jobConfiguration.Input.HourlyTimeRanges[j]; readGranularRangeOfMetricsIntoEntities(entitiesDictionaryByName, jobTimeRange, jobTarget, entityMetricExtractMappingList, DBApplication.ENTITY_FOLDER, DBApplication.ENTITY_TYPE, metricValuesDictionary); } // Save individual metric files and create index of their internal structure foreach (KeyValuePair <string, List <MetricValue> > metricValuesListContainer in metricValuesDictionary) { if (metricValuesListContainer.Value.Count > 0) { List <MetricValue> metricValuesSorted = metricValuesListContainer.Value.OrderBy(o => o.EntityID).ThenBy(o => o.MetricID).ThenBy(o => o.EventTimeStampUtc).ToList(); FileIOHelper.WriteListToCSVFile(metricValuesSorted, new MetricValueReportMap(), FilePathMap.MetricValuesIndexFilePath(jobTarget, DBApplication.ENTITY_FOLDER, metricValuesListContainer.Key)); FileIOHelper.WriteListToCSVFile(metricValuesSorted, new MetricValueReportMap(), FilePathMap.MetricReportPerAppFilePath(jobTarget, DBApplication.ENTITY_FOLDER, metricValuesListContainer.Key)); } } Interlocked.Add(ref numEntitiesTotal, metricValuesDictionary.Keys.Count); } #endregion } ); stepTimingTarget.NumEntities = numEntitiesTotal; #region Combine All for Report CSV // If it is the first one, clear out the combined folder if (reportFolderCleaned == false) { FileIOHelper.DeleteFolder(FilePathMap.MetricsReportFolderPath(jobTarget)); Thread.Sleep(1000); FileIOHelper.CreateFolder(FilePathMap.MetricsReportFolderPath(jobTarget)); reportFolderCleaned = true; } // Combine the generated detailed metric value files foreach (MetricExtractMapping metricExtractMapping in entityMetricExtractMappingList) { switch (metricExtractMapping.EntityType) { case DBApplication.ENTITY_TYPE: FileIOHelper.AppendTwoCSVFiles( FilePathMap.MetricReportFilePath(DBApplication.ENTITY_FOLDER, metricExtractMapping.FolderName, jobTarget), FilePathMap.MetricValuesIndexFilePath(jobTarget, DBApplication.ENTITY_FOLDER, metricExtractMapping.FolderName)); break; default: break; } } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(jobConfiguration) == false) { return(true); } bool reportFolderCleaned = false; // Process each Controller once int i = 0; var controllers = jobConfiguration.Target.GroupBy(t => t.Controller); foreach (var controllerGroup in controllers) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = controllerGroup.ToList()[0]; StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; stepTimingTarget.NumEntities = 0; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Prepare time variables long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From); long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To); long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000); string DEEPLINK_THIS_TIMERANGE = String.Format(DEEPLINK_TIMERANGE_BETWEEN_TIMES, toTimeUnix, fromTimeUnix, differenceInMinutes); #endregion #region Notification Events loggerConsole.Info("Index Notification Events"); List <Event> eventsList = new List <Event>(); JObject notificationsContainer = FileIOHelper.LoadJObjectFromFile(FilePathMap.NotificationsDataFilePath(jobTarget)); if (notificationsContainer != null) { if (isTokenPropertyNull(notificationsContainer, "notifications") == false) { foreach (JObject interestingEvent in notificationsContainer["notifications"]) { if (isTokenPropertyNull(interestingEvent, "notificationData") == false && isTokenPropertyNull(interestingEvent["notificationData"], "affectedEntities") == false) { foreach (JObject affectedEntity in interestingEvent["notificationData"]["affectedEntities"]) { Event @event = new Event(); @event.Controller = jobTarget.Controller; @event.EventID = getLongValueFromJToken(interestingEvent, "id"); @event.OccurredUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(interestingEvent["notificationData"], "time")); try { @event.Occurred = @event.OccurredUtc.ToLocalTime(); } catch { } @event.Type = getStringValueFromJToken(interestingEvent["notificationData"], "eventType"); @event.Severity = getStringValueFromJToken(interestingEvent["notificationData"], "severity"); @event.Summary = getStringValueFromJToken(interestingEvent["notificationData"], "summary"); @event.TriggeredEntityID = getLongValueFromJToken(affectedEntity, "entityId"); @event.TriggeredEntityType = getStringValueFromJToken(affectedEntity, "entityType"); @event.ApplicationID = getLongValueFromJToken(interestingEvent["notificationData"], "applicationId"); @event.TierID = getLongValueFromJToken(interestingEvent["notificationData"], "applicationComponentId"); @event.NodeID = getLongValueFromJToken(interestingEvent["notificationData"], "applicationComponentNodeId"); @event.BTID = getLongValueFromJToken(interestingEvent["notificationData"], "businessTransactionId"); @event.ControllerLink = String.Format(DEEPLINK_CONTROLLER, @event.Controller, DEEPLINK_THIS_TIMERANGE); eventsList.Add(@event); } } } } } loggerConsole.Info("{0} Notification Events", eventsList.Count); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + eventsList.Count; // Sort them eventsList = eventsList.OrderBy(o => o.Type).ThenBy(o => o.Occurred).ThenBy(o => o.Severity).ToList(); FileIOHelper.WriteListToCSVFile <Event>(eventsList, new EventReportMap(), FilePathMap.NotificationsIndexFilePath(jobTarget)); #endregion #region Audit Events loggerConsole.Info("Index Audit Log Events"); List <AuditEvent> auditEventsList = new List <AuditEvent>(); JArray auditEvents = FileIOHelper.LoadJArrayFromFile(FilePathMap.AuditEventsDataFilePath(jobTarget)); if (auditEvents != null) { foreach (JObject interestingEvent in auditEvents) { AuditEvent @event = new AuditEvent(); @event.Controller = jobTarget.Controller; @event.EntityID = getLongValueFromJToken(interestingEvent, "objectId"); @event.EntityType = getStringValueFromJToken(interestingEvent, "objectType"); @event.EntityName = getStringValueFromJToken(interestingEvent, "objectName"); @event.UserName = getStringValueFromJToken(interestingEvent, "userName"); @event.AccountName = getStringValueFromJToken(interestingEvent, "accountName"); @event.LoginType = getStringValueFromJToken(interestingEvent, "securityProviderType"); @event.Action = getStringValueFromJToken(interestingEvent, "action"); @event.EntityID = getLongValueFromJToken(interestingEvent, "objectId"); @event.EntityType = getStringValueFromJToken(interestingEvent, "objectType"); @event.EntityName = getStringValueFromJToken(interestingEvent, "objectName"); @event.OccurredUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(interestingEvent, "timeStamp")); try { @event.Occurred = @event.OccurredUtc.ToLocalTime(); } catch { } auditEventsList.Add(@event); } } loggerConsole.Info("{0} Audit Events", auditEventsList.Count); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + auditEventsList.Count; // Sort them auditEventsList = auditEventsList.OrderBy(o => o.Occurred).ToList(); FileIOHelper.WriteListToCSVFile <AuditEvent>(auditEventsList, new AuditEventReportMap(), FilePathMap.AuditEventsIndexFilePath(jobTarget)); #endregion #region Combine All for Report CSV // If it is the first one, clear out the combined folder if (reportFolderCleaned == false) { FileIOHelper.DeleteFolder(FilePathMap.ControllerEventsReportFolderPath()); Thread.Sleep(1000); FileIOHelper.CreateFolder(FilePathMap.ControllerEventsReportFolderPath()); reportFolderCleaned = true; } // Append all the individual report files into one if (File.Exists(FilePathMap.NotificationsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.NotificationsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.NotificationsReportFilePath(), FilePathMap.NotificationsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.AuditEventsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.AuditEventsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.AuditEventsReportFilePath(), FilePathMap.AuditEventsIndexFilePath(jobTarget)); } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } i++; } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(jobConfiguration) == false) { return(true); } if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_APM) == 0) { return(true); } List <string> listOfControllersAlreadyProcessed = new List <string>(jobConfiguration.Target.Count); bool reportFolderCleaned = false; // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_APM) { continue; } StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Health Rule violations loggerConsole.Info("Index Health Rule Violations"); List <HealthRuleViolationEvent> healthRuleViolationList = new List <HealthRuleViolationEvent>(); long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From); long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To); long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000); string DEEPLINK_THIS_TIMERANGE = String.Format(DEEPLINK_TIMERANGE_BETWEEN_TIMES, toTimeUnix, fromTimeUnix, differenceInMinutes); if (File.Exists(FilePathMap.HealthRuleViolationsDataFilePath(jobTarget))) { JArray healthRuleViolationEvents = FileIOHelper.LoadJArrayFromFile(FilePathMap.HealthRuleViolationsDataFilePath(jobTarget)); if (healthRuleViolationEvents != null) { foreach (JObject interestingEvent in healthRuleViolationEvents) { HealthRuleViolationEvent eventRow = new HealthRuleViolationEvent(); eventRow.Controller = jobTarget.Controller; eventRow.ApplicationName = jobTarget.Application; eventRow.ApplicationID = jobTarget.ApplicationID; eventRow.EventID = (long)interestingEvent["id"]; eventRow.FromUtc = UnixTimeHelper.ConvertFromUnixTimestamp((long)interestingEvent["startTimeInMillis"]); eventRow.From = eventRow.FromUtc.ToLocalTime(); if ((long)interestingEvent["endTimeInMillis"] > 0) { eventRow.ToUtc = UnixTimeHelper.ConvertFromUnixTimestamp((long)interestingEvent["endTimeInMillis"]); eventRow.To = eventRow.FromUtc.ToLocalTime(); } eventRow.Status = interestingEvent["incidentStatus"].ToString(); eventRow.Severity = interestingEvent["severity"].ToString(); eventRow.EventLink = String.Format(DEEPLINK_INCIDENT, eventRow.Controller, eventRow.ApplicationID, eventRow.EventID, interestingEvent["startTimeInMillis"], DEEPLINK_THIS_TIMERANGE);; eventRow.Description = interestingEvent["description"].ToString(); if (interestingEvent["triggeredEntityDefinition"].HasValues == true) { eventRow.HealthRuleID = (int)interestingEvent["triggeredEntityDefinition"]["entityId"]; eventRow.HealthRuleName = interestingEvent["triggeredEntityDefinition"]["name"].ToString(); // TODO the health rule can't be hotlinked to until platform rewrites the screen that opens from Flash eventRow.HealthRuleLink = String.Format(DEEPLINK_HEALTH_RULE, eventRow.Controller, eventRow.ApplicationID, eventRow.HealthRuleID, DEEPLINK_THIS_TIMERANGE); } if (interestingEvent["affectedEntityDefinition"].HasValues == true) { eventRow.EntityID = (int)interestingEvent["affectedEntityDefinition"]["entityId"]; eventRow.EntityName = interestingEvent["affectedEntityDefinition"]["name"].ToString(); string entityType = interestingEvent["affectedEntityDefinition"]["entityType"].ToString(); if (entityTypeStringMapping.ContainsKey(entityType) == true) { eventRow.EntityType = entityTypeStringMapping[entityType]; } else { eventRow.EntityType = entityType; } // Come up with links switch (entityType) { case ENTITY_TYPE_FLOWMAP_APPLICATION: eventRow.EntityLink = String.Format(DEEPLINK_APPLICATION, eventRow.Controller, eventRow.ApplicationID, DEEPLINK_THIS_TIMERANGE); break; case ENTITY_TYPE_FLOWMAP_APPLICATION_MOBILE: eventRow.EntityLink = String.Format(DEEPLINK_APPLICATION_MOBILE, eventRow.Controller, eventRow.ApplicationID, eventRow.EntityID, DEEPLINK_THIS_TIMERANGE); break; case ENTITY_TYPE_FLOWMAP_TIER: eventRow.EntityLink = String.Format(DEEPLINK_TIER, eventRow.Controller, eventRow.ApplicationID, eventRow.EntityID, DEEPLINK_THIS_TIMERANGE); break; case ENTITY_TYPE_FLOWMAP_NODE: eventRow.EntityLink = String.Format(DEEPLINK_NODE, eventRow.Controller, eventRow.ApplicationID, eventRow.EntityID, DEEPLINK_THIS_TIMERANGE); break; case ENTITY_TYPE_FLOWMAP_BUSINESS_TRANSACTION: eventRow.EntityLink = String.Format(DEEPLINK_BUSINESS_TRANSACTION, eventRow.Controller, eventRow.ApplicationID, eventRow.EntityID, DEEPLINK_THIS_TIMERANGE); break; case ENTITY_TYPE_FLOWMAP_BACKEND: eventRow.EntityLink = String.Format(DEEPLINK_BACKEND, eventRow.Controller, eventRow.ApplicationID, eventRow.EntityID, DEEPLINK_THIS_TIMERANGE); break; default: logger.Warn("Unknown entity type {0} in affectedEntityDefinition in health rule violations", entityType); break; } } eventRow.ControllerLink = String.Format(DEEPLINK_CONTROLLER, eventRow.Controller, DEEPLINK_THIS_TIMERANGE); eventRow.ApplicationLink = String.Format(DEEPLINK_APPLICATION, eventRow.Controller, eventRow.ApplicationID, DEEPLINK_THIS_TIMERANGE); healthRuleViolationList.Add(eventRow); } } } loggerConsole.Info("{0} Health Rule Violation events", healthRuleViolationList.Count); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + healthRuleViolationList.Count; // Sort them healthRuleViolationList = healthRuleViolationList.OrderBy(o => o.HealthRuleName).ThenBy(o => o.From).ThenBy(o => o.Severity).ToList(); FileIOHelper.WriteListToCSVFile <HealthRuleViolationEvent>(healthRuleViolationList, new HealthRuleViolationEventReportMap(), FilePathMap.HealthRuleViolationsIndexFilePath(jobTarget)); #endregion #region Events loggerConsole.Info("Index Events"); List <Event> eventsList = new List <Event>(); foreach (string eventType in EVENT_TYPES) { loggerConsole.Info("Type {0} Events", eventType); if (File.Exists(FilePathMap.EventsDataFilePath(jobTarget, eventType))) { JArray events = FileIOHelper.LoadJArrayFromFile(FilePathMap.EventsDataFilePath(jobTarget, eventType)); if (events != null) { foreach (JObject interestingEvent in events) { Event eventRow = new Event(); eventRow.Controller = jobTarget.Controller; eventRow.ApplicationName = jobTarget.Application; eventRow.ApplicationID = jobTarget.ApplicationID; eventRow.EventID = (long)interestingEvent["id"]; eventRow.OccurredUtc = UnixTimeHelper.ConvertFromUnixTimestamp((long)interestingEvent["eventTime"]); eventRow.Occurred = eventRow.OccurredUtc.ToLocalTime(); eventRow.Type = interestingEvent["type"].ToString(); eventRow.SubType = interestingEvent["subType"].ToString(); eventRow.Severity = interestingEvent["severity"].ToString(); eventRow.EventLink = interestingEvent["deepLinkUrl"].ToString(); eventRow.Summary = interestingEvent["summary"].ToString(); if (interestingEvent["triggeredEntity"].HasValues == true) { eventRow.TriggeredEntityID = (long)interestingEvent["triggeredEntity"]["entityId"]; eventRow.TriggeredEntityName = interestingEvent["triggeredEntity"]["name"].ToString(); string entityType = interestingEvent["triggeredEntity"]["entityType"].ToString(); if (entityTypeStringMapping.ContainsKey(entityType) == true) { eventRow.TriggeredEntityType = entityTypeStringMapping[entityType]; } else { eventRow.TriggeredEntityType = entityType; } } foreach (JObject affectedEntity in interestingEvent["affectedEntities"]) { string entityType = affectedEntity["entityType"].ToString(); switch (entityType) { case ENTITY_TYPE_FLOWMAP_APPLICATION: // already have this data break; case ENTITY_TYPE_FLOWMAP_TIER: eventRow.TierID = (int)affectedEntity["entityId"]; eventRow.TierName = affectedEntity["name"].ToString(); break; case ENTITY_TYPE_FLOWMAP_NODE: eventRow.NodeID = (int)affectedEntity["entityId"]; eventRow.NodeName = affectedEntity["name"].ToString(); break; case ENTITY_TYPE_FLOWMAP_MACHINE: eventRow.MachineID = (int)affectedEntity["entityId"]; eventRow.MachineName = affectedEntity["name"].ToString(); break; case ENTITY_TYPE_FLOWMAP_BUSINESS_TRANSACTION: eventRow.BTID = (int)affectedEntity["entityId"]; eventRow.BTName = affectedEntity["name"].ToString(); break; case ENTITY_TYPE_FLOWMAP_HEALTH_RULE: eventRow.TriggeredEntityID = (int)affectedEntity["entityId"]; eventRow.TriggeredEntityType = entityTypeStringMapping[affectedEntity["entityType"].ToString()]; eventRow.TriggeredEntityName = affectedEntity["name"].ToString(); break; default: logger.Warn("Unknown entity type {0} in affectedEntities in events", entityType); break; } } eventRow.ControllerLink = String.Format(DEEPLINK_CONTROLLER, eventRow.Controller, DEEPLINK_THIS_TIMERANGE); eventRow.ApplicationLink = String.Format(DEEPLINK_APPLICATION, eventRow.Controller, eventRow.ApplicationID, DEEPLINK_THIS_TIMERANGE); if (eventRow.TierID != 0) { eventRow.TierLink = String.Format(DEEPLINK_TIER, eventRow.Controller, eventRow.ApplicationID, eventRow.TierID, DEEPLINK_THIS_TIMERANGE); } if (eventRow.NodeID != 0) { eventRow.NodeLink = String.Format(DEEPLINK_NODE, eventRow.Controller, eventRow.ApplicationID, eventRow.NodeID, DEEPLINK_THIS_TIMERANGE); } if (eventRow.BTID != 0) { eventRow.BTLink = String.Format(DEEPLINK_BUSINESS_TRANSACTION, eventRow.Controller, eventRow.ApplicationID, eventRow.BTID, DEEPLINK_THIS_TIMERANGE); } eventsList.Add(eventRow); } } } } // Only output this once per controller if (listOfControllersAlreadyProcessed.Contains(jobTarget.Controller) == false) { listOfControllersAlreadyProcessed.Add(jobTarget.Controller); loggerConsole.Info("Index Notification Events"); JObject notificationsContainer = FileIOHelper.LoadJObjectFromFile(FilePathMap.NotificationsDataFilePath(jobTarget)); if (notificationsContainer != null) { foreach (JObject interestingEvent in notificationsContainer["notifications"]) { if ((long)interestingEvent["notificationData"]["applicationId"] < 0) { foreach (JObject affectedEntity in interestingEvent["notificationData"]["affectedEntities"]) { Event eventRow = new Event(); eventRow.Controller = jobTarget.Controller; eventRow.EventID = (long)interestingEvent["id"]; try { eventRow.OccurredUtc = UnixTimeHelper.ConvertFromUnixTimestamp((long)interestingEvent["notificationData"]["time"]); eventRow.Occurred = eventRow.OccurredUtc.ToLocalTime(); } catch { } try { eventRow.Type = interestingEvent["notificationData"]["eventType"].ToString(); } catch { } try { eventRow.Severity = interestingEvent["notificationData"]["severity"].ToString(); } catch { } try { eventRow.Summary = interestingEvent["notificationData"]["summary"].ToString(); } catch { } try { eventRow.TriggeredEntityID = (long)affectedEntity["entityId"]; } catch { } try { eventRow.TriggeredEntityType = affectedEntity["entityType"].ToString(); } catch { } eventRow.ControllerLink = String.Format(DEEPLINK_CONTROLLER, eventRow.Controller, DEEPLINK_THIS_TIMERANGE); eventsList.Add(eventRow); } } } } } loggerConsole.Info("{0} events", eventsList.Count); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + eventsList.Count; // Sort them eventsList = eventsList.OrderBy(o => o.Type).ThenBy(o => o.Occurred).ThenBy(o => o.Severity).ToList(); FileIOHelper.WriteListToCSVFile <Event>(eventsList, new EventReportMap(), FilePathMap.EventsIndexFilePath(jobTarget)); #endregion #region Audit Events if (File.Exists(FilePathMap.AuditEventsIndexFilePath(jobTarget)) == false) { List <AuditEvent> auditEventsList = new List <AuditEvent>(); loggerConsole.Info("Index Audit Log events"); JArray auditEvents = FileIOHelper.LoadJArrayFromFile(FilePathMap.AuditEventsDataFilePath(jobTarget)); if (auditEvents != null) { foreach (JObject interestingEvent in auditEvents) { AuditEvent eventRow = new AuditEvent(); eventRow.Controller = jobTarget.Controller; //eventRow.ApplicationName = jobTarget.Application; //eventRow.ApplicationID = jobTarget.ApplicationID; try { eventRow.EntityID = (long)interestingEvent["objectId"]; } catch { } try { eventRow.EntityType = interestingEvent["objectType"].ToString(); } catch { } try { eventRow.EntityName = interestingEvent["objectName"].ToString(); } catch { } try { eventRow.UserName = interestingEvent["userName"].ToString(); } catch { } try { eventRow.AccountName = interestingEvent["accountName"].ToString(); } catch { } try { eventRow.LoginType = interestingEvent["securityProviderType"].ToString(); } catch { } try { eventRow.Action = interestingEvent["action"].ToString(); } catch { } try { eventRow.EntityID = (long)interestingEvent["objectId"]; } catch { } try { eventRow.EntityType = interestingEvent["objectType"].ToString(); } catch { } try { eventRow.EntityName = interestingEvent["objectName"].ToString(); } catch { } try { eventRow.OccurredUtc = UnixTimeHelper.ConvertFromUnixTimestamp((long)interestingEvent["timeStamp"]); } catch { } try { eventRow.Occurred = eventRow.OccurredUtc.ToLocalTime(); } catch { } auditEventsList.Add(eventRow); } } auditEventsList = auditEventsList.OrderBy(o => o.Occurred).ToList(); loggerConsole.Info("{0} Audit events", auditEventsList.Count); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + auditEventsList.Count; FileIOHelper.WriteListToCSVFile <AuditEvent>(auditEventsList, new AuditEventReportMap(), FilePathMap.AuditEventsIndexFilePath(jobTarget)); } #endregion #region Application List <APMApplication> applicationList = FileIOHelper.ReadListFromCSVFile <APMApplication>(FilePathMap.ApplicationIndexFilePath(jobTarget), new APMApplicationReportMap()); if (applicationList != null && applicationList.Count > 0) { APMApplication applicationsRow = applicationList[0]; applicationsRow.NumEvents = eventsList.Count; applicationsRow.NumEventsError = eventsList.Count(e => e.Severity == "ERROR"); applicationsRow.NumEventsWarning = eventsList.Count(e => e.Severity == "WARN"); applicationsRow.NumEventsInfo = eventsList.Count(e => e.Severity == "INFO"); applicationsRow.NumHRViolations = healthRuleViolationList.Count; applicationsRow.NumHRViolationsCritical = healthRuleViolationList.Count(e => e.Severity == "CRITICAL"); applicationsRow.NumHRViolationsWarning = healthRuleViolationList.Count(e => e.Severity == "WARNING"); applicationsRow.Duration = (int)(jobConfiguration.Input.TimeRange.To - jobConfiguration.Input.TimeRange.From).Duration().TotalMinutes; applicationsRow.From = jobConfiguration.Input.TimeRange.From.ToLocalTime(); applicationsRow.To = jobConfiguration.Input.TimeRange.To.ToLocalTime(); applicationsRow.FromUtc = jobConfiguration.Input.TimeRange.From; applicationsRow.ToUtc = jobConfiguration.Input.TimeRange.To; if (applicationsRow.NumEvents > 0 || applicationsRow.NumHRViolations > 0) { applicationsRow.HasActivity = true; } FileIOHelper.WriteListToCSVFile(applicationList, new ApplicationEventReportMap(), FilePathMap.ApplicationEventsIndexFilePath(jobTarget)); } #endregion #region Combine All for Report CSV // If it is the first one, clear out the combined folder if (reportFolderCleaned == false) { FileIOHelper.DeleteFolder(FilePathMap.EventsReportFolderPath()); Thread.Sleep(1000); FileIOHelper.CreateFolder(FilePathMap.EventsReportFolderPath()); reportFolderCleaned = true; } // Append all the individual application files into one if (File.Exists(FilePathMap.ApplicationEventsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.ApplicationEventsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.ApplicationEventsReportFilePath(), FilePathMap.ApplicationEventsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.HealthRuleViolationsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.HealthRuleViolationsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.HealthRuleViolationsReportFilePath(), FilePathMap.HealthRuleViolationsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.EventsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.EventsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.EventsReportFilePath(), FilePathMap.EventsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.AuditEventsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.AuditEventsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.AuditEventsReportFilePath(), FilePathMap.AuditEventsIndexFilePath(jobTarget)); } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(jobConfiguration) == false) { return(true); } if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_WEB) == 0) { return(true); } bool reportFolderCleaned = false; // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_WEB) { continue; } StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Preload list of detected entities // For later cross-reference List <ControllerApplication> controllerApplicationList = FileIOHelper.ReadListFromCSVFile <ControllerApplication>(FilePathMap.ControllerApplicationsIndexFilePath(jobTarget), new ControllerApplicationReportMap()); #endregion #region Application Summary WEBApplicationConfiguration applicationConfiguration = new WEBApplicationConfiguration(); loggerConsole.Info("Application Summary"); applicationConfiguration.Controller = jobTarget.Controller; applicationConfiguration.ControllerLink = String.Format(DEEPLINK_CONTROLLER, applicationConfiguration.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); applicationConfiguration.ApplicationName = jobTarget.Application; applicationConfiguration.ApplicationID = jobTarget.ApplicationID; applicationConfiguration.ApplicationLink = String.Format(DEEPLINK_APM_APPLICATION, applicationConfiguration.Controller, applicationConfiguration.ApplicationID, DEEPLINK_TIMERANGE_LAST_15_MINUTES); if (controllerApplicationList != null) { ControllerApplication controllerApplication = controllerApplicationList.Where(a => a.Type == APPLICATION_TYPE_WEB && a.ApplicationID == applicationConfiguration.ApplicationID).FirstOrDefault(); if (controllerApplication != null) { applicationConfiguration.ApplicationDescription = controllerApplication.Description; } } // Application Key JObject appKeyObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.WEBApplicationKeyDataFilePath(jobTarget)); if (appKeyObject != null) { applicationConfiguration.ApplicationKey = getStringValueFromJToken(appKeyObject, "appKey"); } // Instrumentation Options JObject instrumentationObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.WEBAgentConfigDataFilePath(jobTarget)); if (instrumentationObject != null) { applicationConfiguration.IsXsccEnabled = getBoolValueFromJToken(instrumentationObject, "enableXssc"); applicationConfiguration.HostOption = getIntValueFromJToken(instrumentationObject, "hostOption"); applicationConfiguration.AgentHTTP = getStringValueFromJToken(instrumentationObject, "jsAgentUrlHttp"); applicationConfiguration.AgentHTTPS = getStringValueFromJToken(instrumentationObject, "jsAgentUrlHttps"); applicationConfiguration.GeoHTTP = getStringValueFromJToken(instrumentationObject, "geoUrlHttp"); applicationConfiguration.GeoHTTPS = getStringValueFromJToken(instrumentationObject, "geoUrlHttps"); applicationConfiguration.BeaconHTTP = getStringValueFromJToken(instrumentationObject, "beaconUrlHttp"); applicationConfiguration.BeaconHTTPS = getStringValueFromJToken(instrumentationObject, "beaconUrlHttps"); applicationConfiguration.AgentCode = getStringValueFromJToken(instrumentationObject, "codeSnippet"); } // Monitoring State string monitoringState = FileIOHelper.ReadFileFromPath(FilePathMap.WEBApplicationMonitoringStateDataFilePath(jobTarget)); if (monitoringState != String.Empty) { bool parsedBool = false; Boolean.TryParse(monitoringState, out parsedBool); applicationConfiguration.IsEnabled = parsedBool; } // Error Detection JObject errorDetectionRulesObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.WEBAgentErrorRulesDataFilePath(jobTarget)); if (errorDetectionRulesObject != null) { applicationConfiguration.IsJSErrorEnabled = getBoolValueFromJToken(errorDetectionRulesObject, "javaScriptErrorCaptureEnabled"); applicationConfiguration.IsAJAXErrorEnabled = getBoolValueFromJToken(errorDetectionRulesObject, "ajaxRequestErrorCaptureEnabled"); applicationConfiguration.IgnoreJSErrors = getStringValueOfObjectFromJToken(errorDetectionRulesObject, "ignoreJavaScriptErrorConfigRules", true); applicationConfiguration.IgnorePageNames = getStringValueOfObjectFromJToken(errorDetectionRulesObject, "ignorePageNames", true); applicationConfiguration.IgnoreURLs = getStringValueOfObjectFromJToken(errorDetectionRulesObject, "ignoreUrls", true); } // Page Settings JObject pageSettingsObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.WEBAgentPageSettingsRulesDataFilePath(jobTarget)); if (pageSettingsObject != null) { if (isTokenPropertyNull(pageSettingsObject, "thresholds") == false) { applicationConfiguration.SlowThresholdType = getStringValueFromJToken(pageSettingsObject["thresholds"]["slowThreshold"], "type"); applicationConfiguration.SlowThreshold = getIntValueFromJToken(pageSettingsObject["thresholds"]["slowThreshold"], "value"); applicationConfiguration.VerySlowThresholdType = getStringValueFromJToken(pageSettingsObject["thresholds"]["verySlowThreshold"], "type"); applicationConfiguration.VerySlowThreshold = getIntValueFromJToken(pageSettingsObject["thresholds"]["verySlowThreshold"], "value"); applicationConfiguration.StallThresholdType = getStringValueFromJToken(pageSettingsObject["thresholds"]["stallThreshold"], "type"); applicationConfiguration.StallThreshold = getIntValueFromJToken(pageSettingsObject["thresholds"]["stallThreshold"], "value"); } applicationConfiguration.Percentiles = getStringValueOfObjectFromJToken(pageSettingsObject, "percentileMetrics", true); applicationConfiguration.SessionTimeout = getIntValueFromJToken(pageSettingsObject["sessionsMonitor"], "sessionTimeoutMins"); applicationConfiguration.IsIPDisplayed = getBoolValueFromJToken(pageSettingsObject, "ipAddressDisplayed"); applicationConfiguration.EnableSlowSnapshots = getBoolValueFromJToken(pageSettingsObject["eventPolicy"], "enableSlowSnapshotCollection"); applicationConfiguration.EnablePeriodicSnapshots = getBoolValueFromJToken(pageSettingsObject["eventPolicy"], "enablePeriodicSnapshotCollection"); applicationConfiguration.EnableErrorSnapshots = getBoolValueFromJToken(pageSettingsObject["eventPolicy"], "enableErrorSnapshotCollection"); } #endregion #region Rules of all kinds loggerConsole.Info("Rules"); #region Page Rules List <WEBPageDetectionRule> pageDetectionRulesList = new List <WEBPageDetectionRule>(1024); JObject pageRulesObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.WEBAgentPageRulesDataFilePath(jobTarget)); if (pageRulesObject != null) { if (isTokenPropertyNull(pageRulesObject, "customNamingIncludeRules") == false) { JArray includeRulesArray = (JArray)pageRulesObject["customNamingIncludeRules"]; foreach (JObject includeRuleObject in includeRulesArray) { WEBPageDetectionRule webPageDetectionRule = fillWebPageDetectionRule(includeRuleObject, jobTarget); if (webPageDetectionRule != null) { webPageDetectionRule.DetectionType = "INCLUDE"; webPageDetectionRule.EntityCategory = "Pages&IFrames"; pageDetectionRulesList.Add(webPageDetectionRule); } } } if (isTokenPropertyNull(pageRulesObject, "customNamingExcludeRules") == false) { JArray excludeRulesArray = (JArray)pageRulesObject["customNamingExcludeRules"]; foreach (JObject excludeRuleObject in excludeRulesArray) { WEBPageDetectionRule webPageDetectionRule = fillWebPageDetectionRule(excludeRuleObject, jobTarget); if (webPageDetectionRule != null) { webPageDetectionRule.DetectionType = "EXCLUDE"; webPageDetectionRule.EntityCategory = "Pages&IFrames"; pageDetectionRulesList.Add(webPageDetectionRule); } } } } #endregion #region AJAX Rules JObject ajaxRulesObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.WEBAgentAjaxRulesDataFilePath(jobTarget)); if (ajaxRulesObject != null) { if (isTokenPropertyNull(ajaxRulesObject, "customNamingIncludeRules") == false) { JArray includeRulesArray = (JArray)ajaxRulesObject["customNamingIncludeRules"]; foreach (JObject includeRuleObject in includeRulesArray) { WEBPageDetectionRule webPageDetectionRule = fillWebPageDetectionRule(includeRuleObject, jobTarget); if (webPageDetectionRule != null) { webPageDetectionRule.DetectionType = "INCLUDE"; webPageDetectionRule.EntityCategory = "Ajax"; pageDetectionRulesList.Add(webPageDetectionRule); } } } if (isTokenPropertyNull(ajaxRulesObject, "customNamingExcludeRules") == false) { JArray excludeRulesArray = (JArray)ajaxRulesObject["customNamingExcludeRules"]; foreach (JObject excludeRuleObject in excludeRulesArray) { WEBPageDetectionRule webPageDetectionRule = fillWebPageDetectionRule(excludeRuleObject, jobTarget); if (webPageDetectionRule != null) { webPageDetectionRule.DetectionType = "EXCLUDE"; webPageDetectionRule.EntityCategory = "Ajax"; pageDetectionRulesList.Add(webPageDetectionRule); } } } if (isTokenPropertyNull(ajaxRulesObject, "eventServiceIncludeRules") == false) { JArray includeRulesArray = (JArray)ajaxRulesObject["eventServiceIncludeRules"]; foreach (JObject includeRuleObject in includeRulesArray) { WEBPageDetectionRule webPageDetectionRule = fillWebPageDetectionRule(includeRuleObject, jobTarget); if (webPageDetectionRule != null) { webPageDetectionRule.DetectionType = "INCLUDE"; webPageDetectionRule.EntityCategory = "AjaxEventsSvc"; pageDetectionRulesList.Add(webPageDetectionRule); } } } if (isTokenPropertyNull(ajaxRulesObject, "eventServiceExcludeRules") == false) { JArray excludeRulesArray = (JArray)ajaxRulesObject["eventServiceExcludeRules"]; foreach (JObject excludeRuleObject in excludeRulesArray) { WEBPageDetectionRule webPageDetectionRule = fillWebPageDetectionRule(excludeRuleObject, jobTarget); if (webPageDetectionRule != null) { webPageDetectionRule.DetectionType = "EXCLUDE"; webPageDetectionRule.EntityCategory = "AjaxEventsSvc"; pageDetectionRulesList.Add(webPageDetectionRule); } } } } #endregion #region Virtual Page Rules JObject virtualPageRulesObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.WEBAgentVirtualPageRulesDataFilePath(jobTarget)); if (virtualPageRulesObject != null) { if (isTokenPropertyNull(virtualPageRulesObject, "customNamingIncludeRules") == false) { JArray includeRulesArray = (JArray)virtualPageRulesObject["customNamingIncludeRules"]; foreach (JObject includeRuleObject in includeRulesArray) { WEBPageDetectionRule webPageDetectionRule = fillWebPageDetectionRule(includeRuleObject, jobTarget); if (webPageDetectionRule != null) { webPageDetectionRule.DetectionType = "INCLUDE"; webPageDetectionRule.EntityCategory = "VirtualPage"; pageDetectionRulesList.Add(webPageDetectionRule); } } } if (isTokenPropertyNull(virtualPageRulesObject, "customNamingExcludeRules") == false) { JArray excludeRulesArray = (JArray)virtualPageRulesObject["customNamingExcludeRules"]; foreach (JObject excludeRuleObject in excludeRulesArray) { WEBPageDetectionRule webPageDetectionRule = fillWebPageDetectionRule(excludeRuleObject, jobTarget); if (webPageDetectionRule != null) { webPageDetectionRule.DetectionType = "EXCLUDE"; webPageDetectionRule.EntityCategory = "VirtualPage"; pageDetectionRulesList.Add(webPageDetectionRule); } } } } #endregion // Sort them pageDetectionRulesList = pageDetectionRulesList.OrderBy(o => o.EntityCategory).ThenBy(o => o.DetectionType).ThenBy(o => o.Priority).ToList(); FileIOHelper.WriteListToCSVFile(pageDetectionRulesList, new WEBPageDetectionRuleReportMap(), FilePathMap.WEBAgentPageAjaxVirtualPageRulesIndexFilePath(jobTarget)); loggerConsole.Info("Completed {0} Rules", pageDetectionRulesList.Count); #endregion #region Synthetic Jobs loggerConsole.Info("Synthetic Jobs"); List <WEBSyntheticJobDefinition> syntheticJobDefinitionsList = null; JObject syntheticJobsObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.WEBSyntheticJobsDataFilePath(jobTarget)); if (syntheticJobsObject != null) { if (isTokenPropertyNull(syntheticJobsObject, "jobListDatas") == false) { JArray syntheticJobsArray = (JArray)syntheticJobsObject["jobListDatas"]; syntheticJobDefinitionsList = new List <WEBSyntheticJobDefinition>(syntheticJobsArray.Count); foreach (JObject syntheticJobObject in syntheticJobsArray) { if (isTokenPropertyNull(syntheticJobObject, "config") == false) { JObject syntheticJobConfigObject = (JObject)syntheticJobObject["config"]; WEBSyntheticJobDefinition syntheticJobDefinition = new WEBSyntheticJobDefinition(); syntheticJobDefinition.Controller = jobTarget.Controller; syntheticJobDefinition.ControllerLink = String.Format(DEEPLINK_CONTROLLER, jobTarget.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); syntheticJobDefinition.ApplicationName = jobTarget.Application; syntheticJobDefinition.ApplicationID = jobTarget.ApplicationID; syntheticJobDefinition.ApplicationLink = String.Format(DEEPLINK_APM_APPLICATION, jobTarget.Controller, jobTarget.ApplicationID, DEEPLINK_TIMERANGE_LAST_15_MINUTES); syntheticJobDefinition.JobName = getStringValueFromJToken(syntheticJobConfigObject, "description"); syntheticJobDefinition.JobID = getStringValueFromJToken(syntheticJobConfigObject, "id"); syntheticJobDefinition.IsUserEnabled = getBoolValueFromJToken(syntheticJobConfigObject, "userEnabled"); syntheticJobDefinition.IsSystemEnabled = getBoolValueFromJToken(syntheticJobConfigObject, "systemEnabled"); syntheticJobDefinition.FailOnError = getBoolValueFromJToken(syntheticJobConfigObject, "failOnPageError"); syntheticJobDefinition.IsPrivateAgent = getBoolValueFromJToken(syntheticJobObject, "hasPrivateAgent"); syntheticJobDefinition.RateUnit = getStringValueFromJToken(syntheticJobConfigObject["rate"], "unit"); syntheticJobDefinition.Rate = getIntValueFromJToken(syntheticJobConfigObject["rate"], "value"); syntheticJobDefinition.Timeout = getIntValueFromJToken(syntheticJobConfigObject, "timeoutSeconds"); syntheticJobDefinition.Days = getStringValueOfObjectFromJToken(syntheticJobConfigObject, "daysOfWeek", true); syntheticJobDefinition.Browsers = getStringValueOfObjectFromJToken(syntheticJobConfigObject, "browserCodes", true); syntheticJobDefinition.Locations = getStringValueOfObjectFromJToken(syntheticJobConfigObject, "locationCodes", true); if (syntheticJobDefinition.Locations.Length > 0) { syntheticJobDefinition.NumLocations = ((JArray)syntheticJobConfigObject["locationCodes"]).Count(); } syntheticJobDefinition.ScheduleMode = getStringValueFromJToken(syntheticJobConfigObject, "scheduleMode"); syntheticJobDefinition.URL = getStringValueFromJToken(syntheticJobConfigObject, "url"); syntheticJobDefinition.Script = getStringValueFromJToken(syntheticJobConfigObject["script"], "script"); if (syntheticJobDefinition.URL.Length > 0) { syntheticJobDefinition.JobType = "URL"; } else { syntheticJobDefinition.JobType = "SCRIPT"; } syntheticJobDefinition.Network = getStringValueOfObjectFromJToken(syntheticJobConfigObject, "networkProfile", false); syntheticJobDefinition.Config = getStringValueOfObjectFromJToken(syntheticJobConfigObject, "composableConfig", false); syntheticJobDefinition.PerfCriteria = getStringValueOfObjectFromJToken(syntheticJobConfigObject, "performanceCriteria", false); syntheticJobDefinition.CreatedOnUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(syntheticJobConfigObject, "created")); try { syntheticJobDefinition.CreatedOn = syntheticJobDefinition.CreatedOnUtc.ToLocalTime(); } catch { } syntheticJobDefinition.UpdatedOnUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(syntheticJobConfigObject, "updated")); try { syntheticJobDefinition.UpdatedOn = syntheticJobDefinition.UpdatedOnUtc.ToLocalTime(); } catch { } syntheticJobDefinitionsList.Add(syntheticJobDefinition); } } // Sort them syntheticJobDefinitionsList = syntheticJobDefinitionsList.OrderBy(o => o.JobName).ToList(); FileIOHelper.WriteListToCSVFile(syntheticJobDefinitionsList, new WEBSyntheticJobDefinitionReportMap(), FilePathMap.WEBSyntheticJobsIndexFilePath(jobTarget)); loggerConsole.Info("Completed {0} Synthetic Jobs", syntheticJobDefinitionsList.Count); } } #endregion #region Application Settings if (pageDetectionRulesList != null) { applicationConfiguration.NumPageRulesInclude = pageDetectionRulesList.Count(r => r.EntityCategory == "Pages&IFrames" && r.DetectionType == "INCLUDE"); applicationConfiguration.NumPageRulesExclude = pageDetectionRulesList.Count(r => r.EntityCategory == "Pages&IFrames" && r.DetectionType == "EXCLUDE"); applicationConfiguration.NumAJAXRulesInclude = pageDetectionRulesList.Count(r => r.EntityCategory == "Ajax" && r.DetectionType == "INCLUDE"); applicationConfiguration.NumAJAXRulesExclude = pageDetectionRulesList.Count(r => r.EntityCategory == "Ajax" && r.DetectionType == "EXCLUDE"); applicationConfiguration.NumVirtPageRulesInclude = pageDetectionRulesList.Count(r => r.EntityCategory == "VirtualPage" && r.DetectionType == "INCLUDE"); applicationConfiguration.NumVirtPageRulesExclude = pageDetectionRulesList.Count(r => r.EntityCategory == "VirtualPage" && r.DetectionType == "EXCLUDE"); } if (syntheticJobDefinitionsList != null) { applicationConfiguration.NumSyntheticJobs = syntheticJobDefinitionsList.Count; } List <WEBApplicationConfiguration> applicationConfigurationsList = new List <WEBApplicationConfiguration>(1); applicationConfigurationsList.Add(applicationConfiguration); FileIOHelper.WriteListToCSVFile(applicationConfigurationsList, new WEBApplicationConfigurationReportMap(), FilePathMap.WEBApplicationConfigurationIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + applicationConfigurationsList.Count; #endregion #region Combine All for Report CSV // If it is the first one, clear out the combined folder if (reportFolderCleaned == false) { FileIOHelper.DeleteFolder(FilePathMap.WEBConfigurationReportFolderPath()); Thread.Sleep(1000); FileIOHelper.CreateFolder(FilePathMap.WEBConfigurationReportFolderPath()); reportFolderCleaned = true; } // Append all the individual report files into one if (File.Exists(FilePathMap.WEBApplicationConfigurationIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.WEBApplicationConfigurationIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.WEBApplicationConfigurationReportFilePath(), FilePathMap.WEBApplicationConfigurationIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.WEBAgentPageAjaxVirtualPageRulesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.WEBAgentPageAjaxVirtualPageRulesIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.WEBAgentPageAjaxVirtualPageRulesReportFilePath(), FilePathMap.WEBAgentPageAjaxVirtualPageRulesIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.WEBSyntheticJobsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.WEBSyntheticJobsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.WEBSyntheticJobsReportFilePath(), FilePathMap.WEBSyntheticJobsIndexFilePath(jobTarget)); } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(jobConfiguration) == false) { return(true); } if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_BIQ) == 0) { return(true); } bool reportFolderCleaned = false; // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_BIQ) { continue; } StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Saved Searches loggerConsole.Info("Saved Searches and Their Widgets"); List <BIQSearch> biqSearchesList = null; List <BIQWidget> biqWidgetsList = null; JArray savedSearchesArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.BIQSearchesDataFilePath(jobTarget)); if (savedSearchesArray != null) { biqSearchesList = new List <BIQSearch>(savedSearchesArray.Count); biqWidgetsList = new List <BIQWidget>(savedSearchesArray.Count * 8); foreach (JObject savedSearchObject in savedSearchesArray) { BIQSearch search = new BIQSearch(); search.Controller = jobTarget.Controller; search.ControllerLink = String.Format(DEEPLINK_CONTROLLER, jobTarget.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); search.ApplicationName = jobTarget.Application; search.ApplicationID = jobTarget.ApplicationID; search.ApplicationLink = String.Format(DEEPLINK_BIQ_APPLICATION, search.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); search.SearchName = getStringValueFromJToken(savedSearchObject, "searchName"); search.InternalName = getStringValueFromJToken(savedSearchObject, "name"); search.Description = getStringValueFromJToken(savedSearchObject, "searchDescription"); search.SearchType = getStringValueFromJToken(savedSearchObject, "searchType"); search.SearchMode = getStringValueFromJToken(savedSearchObject, "searchMode"); search.ViewMode = getStringValueFromJToken(savedSearchObject, "viewMode"); search.Visualization = getStringValueFromJToken(savedSearchObject, "visualization"); search.SearchID = getLongValueFromJToken(savedSearchObject, "id"); search.SearchLink = String.Format(DEEPLINK_BIQ_SEARCH, search.Controller, search.SearchID, DEEPLINK_TIMERANGE_LAST_15_MINUTES); search.CreatedBy = getStringValueFromJToken(savedSearchObject, "createdBy"); search.CreatedOnUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(savedSearchObject, "createdOn")); try { search.CreatedOn = search.CreatedOnUtc.ToLocalTime(); } catch { } search.UpdatedBy = getStringValueFromJToken(savedSearchObject, "modifiedBy"); search.UpdatedOnUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(savedSearchObject, "modifiedOn")); try { search.UpdatedOn = search.UpdatedOnUtc.ToLocalTime(); } catch { } if (isTokenPropertyNull(savedSearchObject, "adqlQueries") == false) { try { if (savedSearchObject["adqlQueries"].Count() == 0) { search.Query = String.Empty; } else if (savedSearchObject["adqlQueries"].Count() == 1) { search.Query = savedSearchObject["adqlQueries"][0].ToString(); } else { search.Query = getStringValueOfObjectFromJToken(savedSearchObject, "adqlQueries", false); } } catch { } } if (search.Query.Length > 0) { Regex regexVersion = new Regex(@"(?i).*FROM\s(\S*)\s?.*", RegexOptions.IgnoreCase); Match match = regexVersion.Match(search.Query); if (match != null) { if (match.Groups.Count > 1) { search.DataSource = match.Groups[1].Value; } } } if (isTokenPropertyNull(savedSearchObject, "widgets") == false) { foreach (JObject searchWidget in savedSearchObject["widgets"]) { BIQWidget widget = new BIQWidget(); widget.Controller = search.Controller; widget.ControllerLink = search.ControllerLink; widget.ApplicationName = search.ApplicationName; widget.ApplicationID = search.ApplicationID; widget.ApplicationLink = search.ApplicationLink; widget.SearchName = search.SearchName; widget.SearchType = search.SearchType; widget.SearchMode = search.SearchMode; widget.SearchID = search.SearchID; widget.SearchLink = search.SearchLink; widget.InternalName = getStringValueFromJToken(searchWidget, "name"); widget.WidgetID = getLongValueFromJToken(searchWidget, "id"); if (isTokenPropertyNull(searchWidget, "adqlQueries") == false) { try { if (searchWidget["adqlQueries"].Count() == 0) { widget.Query = String.Empty; } else if (searchWidget["adqlQueries"].Count() == 1) { widget.Query = searchWidget["adqlQueries"][0].ToString(); } else { widget.Query = getStringValueOfObjectFromJToken(searchWidget, "adqlQueries", false); } } catch { } } if (widget.Query.Length > 0) { Regex regexVersion = new Regex(@"(?i).*FROM\s(\S*)\s?.*", RegexOptions.IgnoreCase); Match match = regexVersion.Match(widget.Query); if (match != null) { if (match.Groups.Count > 1) { widget.DataSource = match.Groups[1].Value; } } } if (isTokenPropertyNull(searchWidget, "properties") == false) { JObject searchWidgetPropertiesObject = (JObject)searchWidget["properties"]; widget.WidgetName = getStringValueFromJToken(searchWidgetPropertiesObject, "title"); widget.LegendLayout = getStringValueFromJToken(searchWidgetPropertiesObject, "legendsLayout"); widget.WidgetType = getStringValueFromJToken(searchWidgetPropertiesObject, "type"); widget.Resolution = getStringValueFromJToken(searchWidgetPropertiesObject, "resolution"); widget.Width = getIntValueFromJToken(searchWidgetPropertiesObject, "sizeX"); widget.Height = getIntValueFromJToken(searchWidgetPropertiesObject, "sizeY"); widget.MinWidth = getIntValueFromJToken(searchWidgetPropertiesObject, "minSizeX"); widget.MinHeight = getIntValueFromJToken(searchWidgetPropertiesObject, "minSizeY"); widget.Column = getIntValueFromJToken(searchWidgetPropertiesObject, "col"); widget.Row = getIntValueFromJToken(searchWidgetPropertiesObject, "row"); widget.IsStacking = getBoolValueFromJToken(searchWidgetPropertiesObject, "isStackingEnabled"); widget.IsDrilledDown = getBoolValueFromJToken(searchWidgetPropertiesObject, "isDrilledDown"); widget.FontSize = getIntValueFromJToken(searchWidgetPropertiesObject, "fontSize"); widget.Color = getIntValueFromJToken(searchWidgetPropertiesObject, "color").ToString("X6"); widget.BackgroundColor = getIntValueFromJToken(searchWidgetPropertiesObject, "backgroundColor").ToString("X6"); } if (isTokenPropertyNull(searchWidget, "timeRangeSpecifier") == false) { JObject searchWidgetTimeRangeObject = (JObject)searchWidget["timeRangeSpecifier"]; widget.TimeRangeType = getStringValueFromJToken(searchWidgetTimeRangeObject, "type"); widget.TimeRangeDuration = getIntValueFromJToken(searchWidgetTimeRangeObject, "durationInMinutes"); if (isTokenPropertyNull(searchWidgetTimeRangeObject, "timeRange") == false) { widget.StartTimeUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(searchWidgetTimeRangeObject["timeRange"], "startTime")); try { widget.StartTime = widget.StartTimeUtc.ToLocalTime(); } catch { } widget.EndTimeUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(searchWidgetTimeRangeObject["timeRange"], "endTime")); try { widget.EndTime = widget.EndTimeUtc.ToLocalTime(); } catch { } } } search.NumWidgets++; biqWidgetsList.Add(widget); } } biqSearchesList.Add(search); } // Sort them biqSearchesList = biqSearchesList.OrderBy(o => o.SearchName).ToList(); FileIOHelper.WriteListToCSVFile(biqSearchesList, new BIQSearchReportMap(), FilePathMap.BIQSearchesIndexFilePath(jobTarget)); biqWidgetsList = biqWidgetsList.OrderBy(o => o.SearchName).ThenBy(o => o.WidgetName).ToList(); FileIOHelper.WriteListToCSVFile(biqWidgetsList, new BIQWidgetReportMap(), FilePathMap.BIQWidgetsIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + biqSearchesList.Count; } #endregion #region Saved Metrics loggerConsole.Info("Saved Metrics"); List <BIQMetric> biqMetricsList = null; JArray savedMetricsArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.BIQMetricsDataFilePath(jobTarget)); if (savedMetricsArray != null) { biqMetricsList = new List <BIQMetric>(savedMetricsArray.Count); foreach (JObject savedMetricObject in savedMetricsArray) { BIQMetric metric = new BIQMetric(); metric.Controller = jobTarget.Controller; metric.ControllerLink = String.Format(DEEPLINK_CONTROLLER, jobTarget.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); metric.ApplicationName = jobTarget.Application; metric.ApplicationID = jobTarget.ApplicationID; metric.ApplicationLink = String.Format(DEEPLINK_BIQ_APPLICATION, metric.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); metric.MetricName = getStringValueFromJToken(savedMetricObject, "queryName"); metric.MetricDescription = getStringValueFromJToken(savedMetricObject, "queryDescription"); metric.Query = getStringValueFromJToken(savedMetricObject, "adqlQueryString"); if (metric.Query.Length > 0) { Regex regexVersion = new Regex(@"(?i).*FROM\s(\S*)\s?.*", RegexOptions.IgnoreCase); Match match = regexVersion.Match(metric.Query); if (match != null) { if (match.Groups.Count > 1) { metric.DataSource = match.Groups[1].Value; } } } metric.EventType = getStringValueFromJToken(savedMetricObject, "eventType"); metric.IsEnabled = getBoolValueFromJToken(savedMetricObject, "queryExecutionEnabled"); metric.LastExecStatus = getStringValueFromJToken(savedMetricObject, "recentExecutionStatus"); metric.LastExecDuration = getIntValueFromJToken(savedMetricObject, "recentQueryExecutionDuration"); metric.SuccessCount = getIntValueFromJToken(savedMetricObject, "totalSuccessCount"); metric.FailureCount = getIntValueFromJToken(savedMetricObject, "totalFailuresCount"); metric.CreatedBy = getStringValueFromJToken(savedMetricObject, "createdBy"); metric.CreatedOnUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(savedMetricObject, "queryCreationTime")); try { metric.CreatedOn = metric.CreatedOnUtc.ToLocalTime(); } catch { } if (isTokenPropertyNull(savedMetricObject, "metricIds") == false) { metric.MetricsIDs = new List <long>(10); foreach (JValue metricIDValue in savedMetricObject["metricIds"]) { long metricID = (long)metricIDValue; metric.MetricsIDs.Add(metricID); } } // Create metric link if (metric.MetricsIDs != null && metric.MetricsIDs.Count > 0) { StringBuilder sb = new StringBuilder(256); foreach (long metricID in metric.MetricsIDs) { if (metricID > 0) { sb.Append(String.Format(DEEPLINK_METRIC_APPLICATION_TARGET_METRIC_ID, metric.ApplicationID, metricID)); sb.Append(","); } } sb.Remove(sb.Length - 1, 1); metric.MetricLink = String.Format(DEEPLINK_METRIC, metric.Controller, metric.ApplicationID, sb.ToString(), DEEPLINK_TIMERANGE_LAST_15_MINUTES); } biqMetricsList.Add(metric); } // Sort them biqMetricsList = biqMetricsList.OrderBy(o => o.MetricName).ToList(); FileIOHelper.WriteListToCSVFile(biqMetricsList, new BIQMetricReportMap(), FilePathMap.BIQMetricsIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + biqMetricsList.Count; } #endregion #region Business Journeys loggerConsole.Info("Business Journeys"); List <BIQBusinessJourney> businessJourneysList = null; JArray businessJourneysArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.BIQBusinessJourneysDataFilePath(jobTarget)); if (businessJourneysArray != null) { businessJourneysList = new List <BIQBusinessJourney>(businessJourneysArray.Count); foreach (JObject businessJourneyObject in businessJourneysArray) { BIQBusinessJourney businessJourney = new BIQBusinessJourney(); businessJourney.Controller = jobTarget.Controller; businessJourney.ControllerLink = String.Format(DEEPLINK_CONTROLLER, jobTarget.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); businessJourney.ApplicationName = jobTarget.Application; businessJourney.ApplicationID = jobTarget.ApplicationID; businessJourney.ApplicationLink = String.Format(DEEPLINK_BIQ_APPLICATION, businessJourney.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); businessJourney.JourneyName = getStringValueFromJToken(businessJourneyObject, "name"); businessJourney.JourneyDescription = getStringValueFromJToken(businessJourneyObject, "description"); businessJourney.JourneyID = getStringValueFromJToken(businessJourneyObject, "id"); businessJourney.State = getStringValueFromJToken(businessJourneyObject, "state"); businessJourney.KeyField = getStringValueFromJToken(businessJourneyObject, "keyFieldName"); businessJourney.IsEnabled = getBoolValueFromJToken(businessJourneyObject, "enabled"); businessJourney.CreatedBy = getStringValueFromJToken(businessJourneyObject, "createdBy"); businessJourney.CreatedOnUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(businessJourneyObject, "createdAt")); try { businessJourney.CreatedOn = businessJourney.CreatedOnUtc.ToLocalTime(); } catch { } businessJourney.UpdatedBy = getStringValueFromJToken(businessJourneyObject, "lastModifiedBy"); businessJourney.UpdatedOnUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(businessJourneyObject, "lastModifiedAt")); try { businessJourney.UpdatedOn = businessJourney.UpdatedOnUtc.ToLocalTime(); } catch { } if (isTokenPropertyNull(businessJourneyObject, "aggregateGraph") == false && isTokenPropertyNull(businessJourneyObject["aggregateGraph"], "root") == false) { StringBuilder sb = new StringBuilder(32 * 10); getStagesRecursive((JObject)businessJourneyObject["aggregateGraph"]["root"], sb); businessJourney.Stages = sb.ToString(); businessJourney.NumStages = businessJourney.Stages.Split('>').Length; } businessJourneysList.Add(businessJourney); } // Sort them businessJourneysList = businessJourneysList.OrderBy(o => o.JourneyName).ToList(); FileIOHelper.WriteListToCSVFile(businessJourneysList, new BIQBusinessJourneyReportMap(), FilePathMap.BIQBusinessJourneysIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + businessJourneysList.Count; } #endregion #region Experience Levels loggerConsole.Info("Experience Levels"); List <BIQExperienceLevel> experienceLevelsList = null; JObject experienceLevelsContainerObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.BIQExperienceLevelsDataFilePath(jobTarget)); if (experienceLevelsContainerObject != null) { if (isTokenPropertyNull(experienceLevelsContainerObject, "items") == false) { JArray experienceLevelsArray = (JArray)experienceLevelsContainerObject["items"]; experienceLevelsList = new List <BIQExperienceLevel>(experienceLevelsArray.Count); foreach (JObject experienceLevelObject in experienceLevelsArray) { BIQExperienceLevel experienceLevel = new BIQExperienceLevel(); experienceLevel.Controller = jobTarget.Controller; experienceLevel.ControllerLink = String.Format(DEEPLINK_CONTROLLER, jobTarget.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); experienceLevel.ApplicationName = jobTarget.Application; experienceLevel.ApplicationID = jobTarget.ApplicationID; experienceLevel.ApplicationLink = String.Format(DEEPLINK_BIQ_APPLICATION, experienceLevel.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); experienceLevel.ExperienceLevelName = getStringValueFromJToken(experienceLevelObject, "configurationName"); experienceLevel.ExperienceLevelID = getStringValueFromJToken(experienceLevelObject, "id"); experienceLevel.DataSource = getStringValueFromJToken(experienceLevelObject, "eventType"); experienceLevel.EventField = getStringValueFromJToken(experienceLevelObject, "eventField"); experienceLevel.Criteria = getStringValueFromJToken(experienceLevelObject, "criteria"); experienceLevel.ThresholdOperator = getStringValueFromJToken(experienceLevelObject, "thresholdOperator"); experienceLevel.ThresholdValue = getStringValueFromJToken(experienceLevelObject, "thresholdValue"); experienceLevel.Period = getStringValueFromJToken(experienceLevelObject, "compliancePeriod"); experienceLevel.Timezone = getStringValueFromJToken(experienceLevelObject, "timeZone"); experienceLevel.IsActive = getBoolValueFromJToken(experienceLevelObject, "active"); experienceLevel.IsIncludeErrors = getBoolValueFromJToken(experienceLevelObject, "includeErrors"); experienceLevel.NormalThreshold = getIntValueFromJToken(experienceLevelObject, "normalThresholdPercent"); experienceLevel.WarningThreshold = getIntValueFromJToken(experienceLevelObject, "warningThresholdPercent"); //experienceLevel.CriticalThreshold = getIntValueFromJToken(experienceLevelObject, "this needs to be calculated somehow"); experienceLevel.StartOnUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(experienceLevelObject, "startDate")); try { experienceLevel.StartOn = experienceLevel.StartOnUtc.ToLocalTime(); } catch { } if (isTokenPropertyNull(experienceLevelObject, "metadata") == false) { JObject experienceLevelMetadataObject = (JObject)experienceLevelObject["metadata"]; experienceLevel.CreatedBy = getStringValueFromJToken(experienceLevelMetadataObject, "createdByResolvedName"); experienceLevel.CreatedOnUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(experienceLevelMetadataObject, "creationTime")); try { experienceLevel.CreatedOn = experienceLevel.CreatedOnUtc.ToLocalTime(); } catch { } experienceLevel.UpdatedBy = getStringValueFromJToken(experienceLevelMetadataObject, "lastModifiedByResolvedName"); experienceLevel.UpdatedOnUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(experienceLevelMetadataObject, "lastModifiedTime")); try { experienceLevel.UpdatedOn = experienceLevel.UpdatedOnUtc.ToLocalTime(); } catch { } } if (isTokenPropertyNull(experienceLevelObject, "exclusionPeriodList") == false) { JArray exclusionPeriodsArray = (JArray)experienceLevelObject["exclusionPeriodList"]; experienceLevel.NumExclusionPeriods = exclusionPeriodsArray.Count; experienceLevel.ExclusionPeriodsRaw = getStringValueOfObjectFromJToken(experienceLevelObject, "exclusionPeriodList", false); } experienceLevelsList.Add(experienceLevel); } // Sort them experienceLevelsList = experienceLevelsList.OrderBy(o => o.ExperienceLevelName).ToList(); FileIOHelper.WriteListToCSVFile(experienceLevelsList, new BIQExperienceLevelReportMap(), FilePathMap.BIQExperienceLevelsIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + businessJourneysList.Count; } } #endregion #region Schema Fields List <BIQSchema> schemasList = new List <BIQSchema>(16); List <BIQField> schemaFieldsList = new List <BIQField>(16 * 32); List <string> analyticsSchemas = new List <string>(BIQ_SCHEMA_TYPES.Count + 10); analyticsSchemas.AddRange(BIQ_SCHEMA_TYPES); // Add custom schemas if any JObject customSchemasContainer = FileIOHelper.LoadJObjectFromFile(FilePathMap.BIQCustomSchemasDataFilePath(jobTarget)); if (customSchemasContainer != null) { JArray customSchemas = JArray.Parse(getStringValueFromJToken(customSchemasContainer, "rawResponse")); foreach (JToken customSchemaToken in customSchemas) { string schemaName = customSchemaToken.ToString(); analyticsSchemas.Add(schemaName); } } // First get known schemas foreach (string schemaName in analyticsSchemas) { loggerConsole.Info("Fields for Schema {0}", schemaName); JArray schemaFieldsArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.BIQSchemaFieldsDataFilePath(jobTarget, schemaName)); if (schemaFieldsArray != null) { List <BIQField> schemaFieldsInThisSchemaList = new List <BIQField>(schemaFieldsArray.Count); foreach (JObject schemaFieldObject in schemaFieldsArray) { BIQField field = new BIQField(); field.Controller = jobTarget.Controller; field.ControllerLink = String.Format(DEEPLINK_CONTROLLER, jobTarget.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); field.ApplicationName = jobTarget.Application; field.ApplicationID = jobTarget.ApplicationID; field.ApplicationLink = String.Format(DEEPLINK_BIQ_APPLICATION, field.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); field.SchemaName = schemaName; field.FieldName = getStringValueFromJToken(schemaFieldObject, "fieldName"); field.FieldType = getStringValueFromJToken(schemaFieldObject, "fieldType"); field.Category = getStringValueFromJToken(schemaFieldObject, "category"); if (isTokenPropertyNull(schemaFieldObject, "parents") == false) { string[] parents = schemaFieldObject["parents"].Select(a => a["fieldName"].ToString()).ToArray(); field.Parents = String.Join(";", parents); field.NumParents = parents.Length; } field.IsSortable = getBoolValueFromJToken(schemaFieldObject, "sortingAllowed"); field.IsAggregatable = getBoolValueFromJToken(schemaFieldObject, "aggregationsAllowed"); field.IsHidden = getBoolValueFromJToken(schemaFieldObject, "hidden"); field.IsDeleted = getBoolValueFromJToken(schemaFieldObject, "softDeleted"); schemaFieldsInThisSchemaList.Add(field); } // Now the schema BIQSchema schema = new BIQSchema(); schema.Controller = jobTarget.Controller; schema.ControllerLink = String.Format(DEEPLINK_CONTROLLER, jobTarget.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); schema.ApplicationName = jobTarget.Application; schema.ApplicationID = jobTarget.ApplicationID; schema.ApplicationLink = String.Format(DEEPLINK_BIQ_APPLICATION, schema.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); schema.SchemaName = schemaName; schema.IsCustom = (BIQ_SCHEMA_TYPES.Contains(schemaName) == false); schema.NumFields = schemaFieldsInThisSchemaList.Count; schema.NumStringFields = schemaFieldsInThisSchemaList.Where(s => s.FieldType == "STRING").Count(); schema.NumIntegerFields = schemaFieldsInThisSchemaList.Where(s => s.FieldType == "INTEGER").Count(); schema.NumLongFields = schemaFieldsInThisSchemaList.Where(s => s.FieldType == "LONG").Count(); schema.NumFloatFields = schemaFieldsInThisSchemaList.Where(s => s.FieldType == "FLOAT").Count(); schema.NumDoubleFields = schemaFieldsInThisSchemaList.Where(s => s.FieldType == "DOUBLE").Count(); schema.NumBooleanFields = schemaFieldsInThisSchemaList.Where(s => s.FieldType == "BOOLEAN").Count(); schema.NumDateFields = schemaFieldsInThisSchemaList.Where(s => s.FieldType == "DATE").Count(); schema.NumObjectFields = schemaFieldsInThisSchemaList.Where(s => s.FieldType == "OBJECT").Count(); schemaFieldsList.AddRange(schemaFieldsInThisSchemaList); schemasList.Add(schema); } } // Sort them schemasList = schemasList.OrderBy(o => o.SchemaName).ToList(); FileIOHelper.WriteListToCSVFile(schemasList, new BIQSchemaReportMap(), FilePathMap.BIQSchemasIndexFilePath(jobTarget)); schemaFieldsList = schemaFieldsList.OrderBy(o => o.SchemaName).ThenBy(o => o.FieldName).ToList(); FileIOHelper.WriteListToCSVFile(schemaFieldsList, new BIQFieldReportMap(), FilePathMap.BIQSchemaFieldsIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + schemaFieldsList.Count; #endregion #region Application loggerConsole.Info("Index Application"); BIQApplication application = new BIQApplication(); application.Controller = jobTarget.Controller; application.ControllerLink = String.Format(DEEPLINK_CONTROLLER, jobTarget.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); application.ApplicationName = jobTarget.Application; application.ApplicationID = jobTarget.ApplicationID; application.ApplicationLink = String.Format(DEEPLINK_BIQ_APPLICATION, application.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); if (biqSearchesList != null) { application.NumSearches = biqSearchesList.Count; application.NumSingleSearches = biqSearchesList.Count(p => p.SearchType == "SINGLE"); application.NumMultiSearches = biqSearchesList.Count(p => p.SearchType == "MULTI"); application.NumLegacySearches = biqSearchesList.Count(p => p.SearchType == "LEGACY42"); } if (biqMetricsList != null) { application.NumSavedMetrics = biqMetricsList.Count; } if (businessJourneysList != null) { application.NumBusinessJourneys = businessJourneysList.Count; } if (experienceLevelsList != null) { application.NumExperienceLevels = experienceLevelsList.Count; } if (schemasList != null) { application.NumSchemas = schemasList.Count; } if (schemaFieldsList != null) { application.NumFields = schemaFieldsList.Count; } List <BIQApplication> applicationsList = new List <BIQApplication>(1); applicationsList.Add(application); FileIOHelper.WriteListToCSVFile(applicationsList, new BIQApplicationReportMap(), FilePathMap.BIQApplicationsIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + 1; #endregion #region Combine All for Report CSV // If it is the first one, clear out the combined folder if (reportFolderCleaned == false) { FileIOHelper.DeleteFolder(FilePathMap.BIQEntitiesReportFolderPath()); Thread.Sleep(1000); FileIOHelper.CreateFolder(FilePathMap.BIQEntitiesReportFolderPath()); reportFolderCleaned = true; } // Append all the individual report files into one if (File.Exists(FilePathMap.BIQApplicationsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.BIQApplicationsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.BIQApplicationsReportFilePath(), FilePathMap.BIQApplicationsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.BIQSearchesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.BIQSearchesIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.BIQSearchesReportFilePath(), FilePathMap.BIQSearchesIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.BIQWidgetsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.BIQWidgetsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.BIQWidgetsReportFilePath(), FilePathMap.BIQWidgetsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.BIQMetricsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.BIQMetricsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.BIQMetricsReportFilePath(), FilePathMap.BIQMetricsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.BIQBusinessJourneysIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.BIQBusinessJourneysIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.BIQBusinessJourneysReportFilePath(), FilePathMap.BIQBusinessJourneysIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.BIQExperienceLevelsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.BIQExperienceLevelsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.BIQExperienceLevelsReportFilePath(), FilePathMap.BIQExperienceLevelsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.BIQSchemasIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.BIQSchemasIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.BIQSchemasReportFilePath(), FilePathMap.BIQSchemasIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.BIQSchemaFieldsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.BIQSchemaFieldsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.BIQSchemaFieldsReportFilePath(), FilePathMap.BIQSchemaFieldsIndexFilePath(jobTarget)); } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } // Let's append all Applications var controllers = jobConfiguration.Target.GroupBy(t => t.Controller); foreach (var controllerGroup in controllers) { } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(programOptions, jobConfiguration) == false) { return(true); } if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_MOBILE) == 0) { logger.Warn("No {0} targets to process", APPLICATION_TYPE_MOBILE); loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_MOBILE); return(true); } #region Template comparisons if (jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE.Controller == BLANK_APPLICATION_CONTROLLER && jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE.Application == BLANK_APPLICATION_MOBILE) { jobConfiguration.Target.Add(jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE); } else { // Check if there is a valid reference application JobTarget jobTargetReferenceApp = jobConfiguration.Target.Where(t => t.Type == APPLICATION_TYPE_MOBILE && String.Compare(t.Controller, jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE.Controller, StringComparison.InvariantCultureIgnoreCase) == 0 && String.Compare(t.Application, jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE.Application, StringComparison.InvariantCultureIgnoreCase) == 0).FirstOrDefault(); if (jobTargetReferenceApp == null) { // No valid reference, fall back to comparing against template logger.Warn("Unable to find reference target {0}, will index default template", jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE); loggerConsole.Warn("Unable to find reference target {0}, will index default template", jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE); jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE.Controller = BLANK_APPLICATION_CONTROLLER; jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE.Application = BLANK_APPLICATION_MOBILE; jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE.Type = APPLICATION_TYPE_MOBILE; jobConfiguration.Target.Add(jobConfiguration.Input.ConfigurationComparisonReferenceMOBILE); } } #endregion bool reportFolderCleaned = false; // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_MOBILE) { continue; } StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Preload list of detected entities // For later cross-reference List <ControllerApplication> controllerApplicationList = FileIOHelper.ReadListFromCSVFile <ControllerApplication>(FilePathMap.ControllerApplicationsIndexFilePath(jobTarget), new ControllerApplicationReportMap()); #endregion #region Application Summary MOBILEApplicationConfiguration applicationConfiguration = new MOBILEApplicationConfiguration(); loggerConsole.Info("Application Summary"); applicationConfiguration.Controller = jobTarget.Controller; applicationConfiguration.ControllerLink = String.Format(DEEPLINK_CONTROLLER, applicationConfiguration.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); applicationConfiguration.ApplicationName = jobTarget.Application; applicationConfiguration.ApplicationID = jobTarget.ApplicationID; applicationConfiguration.ApplicationLink = String.Format(DEEPLINK_APM_APPLICATION, applicationConfiguration.Controller, applicationConfiguration.ApplicationID, DEEPLINK_TIMERANGE_LAST_15_MINUTES); if (controllerApplicationList != null) { ControllerApplication controllerApplication = controllerApplicationList.Where(a => a.Type == APPLICATION_TYPE_MOBILE && a.ApplicationID == jobTarget.ApplicationID).FirstOrDefault(); if (controllerApplication != null) { applicationConfiguration.ApplicationDescription = controllerApplication.Description; } } // Application Key JObject appKeyObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.MOBILEApplicationKeyDataFilePath(jobTarget)); if (appKeyObject != null) { applicationConfiguration.ApplicationKey = getStringValueFromJToken(appKeyObject, "appKey"); } // Monitoring State string monitoringState = FileIOHelper.ReadFileFromPath(FilePathMap.MOBILEApplicationMonitoringStateDataFilePath(jobTarget)); if (monitoringState != String.Empty) { bool parsedBool = false; Boolean.TryParse(monitoringState, out parsedBool); applicationConfiguration.IsEnabled = parsedBool; } // Configuration Settings JObject configurationSettingsObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.MOBILEAgentPageSettingsRulesDataFilePath(jobTarget)); if (configurationSettingsObject != null) { if (isTokenPropertyNull(configurationSettingsObject, "thresholds") == false) { applicationConfiguration.SlowThresholdType = getStringValueFromJToken(configurationSettingsObject["thresholds"]["slowThreshold"], "type"); applicationConfiguration.SlowThreshold = getIntValueFromJToken(configurationSettingsObject["thresholds"]["slowThreshold"], "value"); applicationConfiguration.VerySlowThresholdType = getStringValueFromJToken(configurationSettingsObject["thresholds"]["verySlowThreshold"], "type"); applicationConfiguration.VerySlowThreshold = getIntValueFromJToken(configurationSettingsObject["thresholds"]["verySlowThreshold"], "value"); applicationConfiguration.StallThresholdType = getStringValueFromJToken(configurationSettingsObject["thresholds"]["stallThreshold"], "type"); applicationConfiguration.StallThreshold = getIntValueFromJToken(configurationSettingsObject["thresholds"]["stallThreshold"], "value"); } applicationConfiguration.Percentiles = getStringValueOfObjectFromJToken(configurationSettingsObject, "percentileMetrics", true); applicationConfiguration.SessionTimeout = getIntValueFromJToken(configurationSettingsObject["sessionsMonitor"], "sessionTimeoutMins"); applicationConfiguration.CrashThreshold = getIntValueFromJToken(configurationSettingsObject["crashAlerts"], "threshold"); applicationConfiguration.IsIPDisplayed = getBoolValueFromJToken(configurationSettingsObject, "ipAddressDisplayed"); applicationConfiguration.EnableScreenshot = getBoolValueFromJToken(configurationSettingsObject["agentConfigData"], "enableScreenshot"); applicationConfiguration.AutoScreenshot = getBoolValueFromJToken(configurationSettingsObject["agentConfigData"], "autoScreenshot"); applicationConfiguration.UseCellular = getBoolValueFromJToken(configurationSettingsObject["agentConfigData"], "screenshotUseCellular"); } #endregion #region Network Requests List <MOBILENetworkRequestRule> networkRequestRulesList = new List <MOBILENetworkRequestRule>(128); JObject networkRequestRulesObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.MOBILEAgentNetworkRequestsRulesDataFilePath(jobTarget)); if (networkRequestRulesObject != null) { if (isTokenPropertyNull(networkRequestRulesObject, "customNamingIncludeRules") == false) { JArray includeRulesArray = (JArray)networkRequestRulesObject["customNamingIncludeRules"]; foreach (JObject includeRuleObject in includeRulesArray) { MOBILENetworkRequestRule networkRequestRule = fillNetworkRequestRule(includeRuleObject, jobTarget); if (networkRequestRule != null) { networkRequestRule.DetectionType = "INCLUDE"; networkRequestRulesList.Add(networkRequestRule); } } } if (isTokenPropertyNull(networkRequestRulesObject, "customNamingExcludeRules") == false) { JArray excludeRulesArray = (JArray)networkRequestRulesObject["customNamingExcludeRules"]; foreach (JObject excludeRuleObject in excludeRulesArray) { MOBILENetworkRequestRule networkRequestRule = fillNetworkRequestRule(excludeRuleObject, jobTarget); if (networkRequestRule != null) { networkRequestRule.DetectionType = "EXCLUDE"; networkRequestRulesList.Add(networkRequestRule); } } } } // Sort them networkRequestRulesList = networkRequestRulesList.OrderBy(o => o.DetectionType).ThenBy(o => o.Priority).ToList(); FileIOHelper.WriteListToCSVFile(networkRequestRulesList, new MOBILENetworkRequestRuleReportMap(), FilePathMap.MOBILENetworkRequestRulesIndexFilePath(jobTarget)); loggerConsole.Info("Completed {0} Rules", networkRequestRulesList.Count); #endregion #region Application Settings if (networkRequestRulesList != null) { applicationConfiguration.NumNetworkRulesInclude = networkRequestRulesList.Count(r => r.DetectionType == "INCLUDE"); applicationConfiguration.NumNetworkRulesExclude = networkRequestRulesList.Count(r => r.DetectionType == "EXCLUDE"); } List <MOBILEApplicationConfiguration> applicationConfigurationsList = new List <MOBILEApplicationConfiguration>(1); applicationConfigurationsList.Add(applicationConfiguration); FileIOHelper.WriteListToCSVFile(applicationConfigurationsList, new MOBILEApplicationConfigurationReportMap(), FilePathMap.MOBILEApplicationConfigurationIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + applicationConfigurationsList.Count; #endregion #region Combine All for Report CSV // If it is the first one, clear out the combined folder if (reportFolderCleaned == false) { FileIOHelper.DeleteFolder(FilePathMap.MOBILEConfigurationReportFolderPath()); Thread.Sleep(1000); FileIOHelper.CreateFolder(FilePathMap.WEBConfigurationReportFolderPath()); reportFolderCleaned = true; } // Append all the individual report files into one if (File.Exists(FilePathMap.MOBILEApplicationConfigurationIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.MOBILEApplicationConfigurationIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.MOBILEApplicationConfigurationReportFilePath(), FilePathMap.MOBILEApplicationConfigurationIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.MOBILENetworkRequestRulesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.MOBILENetworkRequestRulesIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.MOBILENetworkRequestRulesReportFilePath(), FilePathMap.MOBILENetworkRequestRulesIndexFilePath(jobTarget)); } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } // Remove all templates from the list jobConfiguration.Target.RemoveAll(t => t.Controller == BLANK_APPLICATION_CONTROLLER); return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(jobConfiguration) == false) { return(true); } bool reportFolderCleaned = false; // Process each Controller once int i = 0; var controllers = jobConfiguration.Target.GroupBy(t => t.Controller); foreach (var controllerGroup in controllers) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = controllerGroup.ToList()[0]; StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; stepTimingTarget.NumEntities = 0; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Users JArray usersArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.UsersDataFilePath(jobTarget)); List <RBACUser> usersList = null; if (usersArray != null && usersArray.Count > 0) { loggerConsole.Info("Index List of Users ({0} entities)", usersArray.Count); usersList = new List <RBACUser>(usersArray.Count); foreach (JToken userToken in usersArray) { RBACUser user = new RBACUser(); user.Controller = jobTarget.Controller; user.UserName = getStringValueFromJToken(userToken, "name"); user.DisplayName = getStringValueFromJToken(userToken, "displayName"); user.Email = getStringValueFromJToken(userToken, "email"); user.SecurityProvider = getStringValueFromJToken(userToken, "securityProviderType"); user.UserID = getLongValueFromJToken(userToken, "id"); user.CreatedBy = getStringValueFromJToken(userToken, "createdBy"); user.CreatedOnUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(userToken, "createdOn")); try { user.CreatedOn = user.CreatedOnUtc.ToLocalTime(); } catch { } user.UpdatedBy = getStringValueFromJToken(userToken, "modifiedBy"); user.UpdatedOnUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(userToken, "modifiedOn")); try { user.UpdatedOn = user.UpdatedOnUtc.ToLocalTime(); } catch { } usersList.Add(user); } // Sort them usersList = usersList.OrderBy(o => o.SecurityProvider).ThenBy(o => o.UserName).ToList(); FileIOHelper.WriteListToCSVFile(usersList, new RBACUserReportMap(), FilePathMap.UsersIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + usersList.Count; } #endregion #region Groups JArray groupsArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.GroupsDataFilePath(jobTarget)); List <RBACGroup> groupsList = null; if (groupsArray != null && groupsArray.Count > 0) { loggerConsole.Info("Index List of Groups ({0} entities)", groupsArray.Count); groupsList = new List <RBACGroup>(groupsArray.Count); foreach (JToken groupToken in groupsArray) { RBACGroup group = new RBACGroup(); group.Controller = jobTarget.Controller; group.GroupName = getStringValueFromJToken(groupToken, "name"); group.Description = getStringValueFromJToken(groupToken, "description"); group.SecurityProvider = getStringValueFromJToken(groupToken, "securityProviderType"); group.GroupID = getLongValueFromJToken(groupToken, "id"); group.CreatedBy = getStringValueFromJToken(groupToken, "createdBy"); group.CreatedOnUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(groupToken, "createdOn")); try { group.CreatedOn = group.CreatedOnUtc.ToLocalTime(); } catch { } group.UpdatedBy = getStringValueFromJToken(groupToken, "modifiedBy"); group.UpdatedOnUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(groupToken, "modifiedOn")); try { group.UpdatedOn = group.UpdatedOnUtc.ToLocalTime(); } catch { } groupsList.Add(group); } // Sort them groupsList = groupsList.OrderBy(o => o.SecurityProvider).ThenBy(o => o.GroupName).ToList(); FileIOHelper.WriteListToCSVFile(groupsList, new RBACGroupReportMap(), FilePathMap.GroupsIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + groupsList.Count; } #endregion #region Roles JArray rolesArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.RolesDataFilePath(jobTarget)); List <RBACRole> rolesList = null; List <RBACPermission> permissionsList = null; List <ControllerApplication> applicationsList = FileIOHelper.ReadListFromCSVFile <ControllerApplication>(FilePathMap.ControllerApplicationsIndexFilePath(jobTarget), new ControllerApplicationReportMap()); if (rolesArray != null && rolesArray.Count > 0) { loggerConsole.Info("Index List of Roles ({0} entities)", rolesArray.Count); rolesList = new List <RBACRole>(rolesArray.Count); permissionsList = new List <RBACPermission>(rolesArray.Count * 32); foreach (JToken roleToken in rolesArray) { RBACRole role = new RBACRole(); role.Controller = jobTarget.Controller; role.RoleName = getStringValueFromJToken(roleToken, "name"); role.Description = getStringValueFromJToken(roleToken, "description"); role.ReadOnly = getBoolValueFromJToken(roleToken, "readonly"); role.RoleID = getLongValueFromJToken(roleToken, "id"); role.CreatedBy = getStringValueFromJToken(roleToken, "createdBy"); role.CreatedOnUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(roleToken, "createdOn")); try { role.CreatedOn = role.CreatedOnUtc.ToLocalTime(); } catch { } role.UpdatedBy = getStringValueFromJToken(roleToken, "modifiedBy"); role.UpdatedOnUtc = UnixTimeHelper.ConvertFromUnixTimestamp(getLongValueFromJToken(roleToken, "modifiedOn")); try { role.UpdatedOn = role.UpdatedOnUtc.ToLocalTime(); } catch { } // Permissions from role detail JObject roleDetailObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.RoleDataFilePath(jobTarget, role.RoleName, role.RoleID)); if (roleDetailObject != null) { foreach (JToken permissionToken in roleDetailObject["permissions"]) { RBACPermission permission = new RBACPermission(); permission.Controller = role.Controller; permission.RoleName = role.RoleName; permission.RoleID = role.RoleID; permission.PermissionName = getStringValueFromJToken(permissionToken, "action"); permission.Allowed = getBoolValueFromJToken(permissionToken, "allowed"); permission.PermissionID = getLongValueFromJToken(permissionToken, "id"); if (isTokenPropertyNull(permissionToken, "affectedEntity") == false) { permission.EntityType = getStringValueFromJToken(permissionToken["affectedEntity"], "entityType"); permission.EntityID = getLongValueFromJToken(permissionToken["affectedEntity"], "entityId"); } // Lookup the application if (permission.EntityType == "APPLICATION" && permission.EntityID != 0) { if (applicationsList != null) { ControllerApplication application = applicationsList.Where(e => e.ApplicationID == permission.EntityID).FirstOrDefault(); if (application != null) { permission.EntityName = application.ApplicationName; } } } else { permission.EntityType = ""; } role.NumPermissions++; permissionsList.Add(permission); } } rolesList.Add(role); } // Sort them rolesList = rolesList.OrderBy(o => o.RoleName).ToList(); FileIOHelper.WriteListToCSVFile(rolesList, new RBACRoleReportMap(), FilePathMap.RolesIndexFilePath(jobTarget)); permissionsList = permissionsList.OrderBy(o => o.RoleName).ThenBy(o => o.EntityName).ThenBy(o => o.PermissionName).ToList(); FileIOHelper.WriteListToCSVFile(permissionsList, new RBACPermissionReportMap(), FilePathMap.PermissionsIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + rolesList.Count; } #endregion #region Groups and Users in Roles List <RBACRoleMembership> roleMembershipsList = new List <RBACRoleMembership>(); // Users in Roles if (usersList != null && rolesList != null) { loggerConsole.Info("Index Users in Roles ({0} entities)", usersList.Count); foreach (RBACUser user in usersList) { JObject userDetailObject = FileIOHelper.LoadJObjectFromFile(FilePathMap.UserDataFilePath(jobTarget, user.UserName, user.UserID)); if (userDetailObject != null) { foreach (JToken roleIDToken in userDetailObject["accountRoleIds"]) { long roleID = (long)roleIDToken; RBACRole role = rolesList.Where(r => r.RoleID == roleID).FirstOrDefault(); if (role != null) { RBACRoleMembership roleMembership = new RBACRoleMembership(); roleMembership.Controller = user.Controller; roleMembership.RoleName = role.RoleName; roleMembership.RoleID = role.RoleID; roleMembership.EntityName = user.UserName; roleMembership.EntityID = user.UserID; roleMembership.EntityType = "User"; roleMembershipsList.Add(roleMembership); } } } } } // Groups in Roles if (groupsList != null && rolesList != null) { loggerConsole.Info("Index Groups in Roles ({0} entities)", groupsList.Count); foreach (RBACGroup group in groupsList) { JObject groupDetailJSON = FileIOHelper.LoadJObjectFromFile(FilePathMap.GroupDataFilePath(jobTarget, group.GroupName, group.GroupID)); if (groupDetailJSON != null) { foreach (JToken roleIDToken in groupDetailJSON["accountRoleIds"]) { long roleID = (long)roleIDToken; RBACRole role = rolesList.Where(r => r.RoleID == roleID).FirstOrDefault(); if (role != null) { RBACRoleMembership roleMembership = new RBACRoleMembership(); roleMembership.Controller = group.Controller; roleMembership.RoleName = role.RoleName; roleMembership.RoleID = role.RoleID; roleMembership.EntityName = group.GroupName; roleMembership.EntityID = group.GroupID; roleMembership.EntityType = "Group"; roleMembershipsList.Add(roleMembership); } } } } } roleMembershipsList = roleMembershipsList.OrderBy(o => o.RoleName).ThenBy(o => o.EntityType).ThenBy(o => o.EntityName).ToList(); FileIOHelper.WriteListToCSVFile(roleMembershipsList, new RBACRoleMembershipReportMap(), FilePathMap.RoleMembershipsIndexFilePath(jobTarget)); #endregion #region Users in Groups List <RBACGroupMembership> groupMembershipsList = new List <RBACGroupMembership>(); if (groupsList != null && usersList != null) { loggerConsole.Info("Index Users in Groups ({0} entities)", groupsList.Count); foreach (RBACGroup group in groupsList) { JArray usersInGroupArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.GroupUsersDataFilePath(jobTarget, group.GroupName, group.GroupID)); if (usersInGroupArray != null) { foreach (JToken userIDToken in usersInGroupArray) { long userID = (long)userIDToken; RBACUser user = usersList.Where(r => r.UserID == userID).FirstOrDefault(); if (user != null) { RBACGroupMembership groupMembership = new RBACGroupMembership(); groupMembership.Controller = group.Controller; groupMembership.GroupName = group.GroupName; groupMembership.GroupID = group.GroupID; groupMembership.UserName = user.UserName; groupMembership.UserID = user.UserID; groupMembershipsList.Add(groupMembership); } } } } } groupMembershipsList = groupMembershipsList.OrderBy(o => o.GroupName).ThenBy(o => o.UserName).ToList(); FileIOHelper.WriteListToCSVFile(groupMembershipsList, new RBACGroupMembershipReportMap(), FilePathMap.GroupMembershipsIndexFilePath(jobTarget)); #endregion #region User Permissions List <RBACUserPermission> userPermissionsList = new List <RBACUserPermission>(); if (roleMembershipsList != null && permissionsList != null) { loggerConsole.Info("Index Users Permissions ({0} entities)", roleMembershipsList.Count); // Scroll through the list of Role memberships foreach (RBACRoleMembership roleMembership in roleMembershipsList) { if (roleMembership.EntityType == "User") { if (usersList != null) { // For User, enumerate permissions associated with this role RBACUser user = usersList.Where(u => u.UserID == roleMembership.EntityID).FirstOrDefault(); if (user != null) { List <RBACPermission> permissionsForRoleList = permissionsList.Where(p => p.RoleID == roleMembership.RoleID).ToList(); if (permissionsForRoleList != null) { foreach (RBACPermission permission in permissionsForRoleList) { RBACUserPermission userPermission = new RBACUserPermission(); userPermission.Controller = user.Controller; userPermission.UserName = user.UserName; userPermission.UserSecurityProvider = user.SecurityProvider; userPermission.UserID = user.UserID; userPermission.RoleName = permission.RoleName; userPermission.RoleID = permission.RoleID; userPermission.PermissionName = permission.PermissionName; userPermission.PermissionID = permission.PermissionID; userPermission.Allowed = permission.Allowed; userPermission.EntityName = permission.EntityName; userPermission.EntityType = permission.EntityType; userPermission.EntityID = permission.EntityID; userPermissionsList.Add(userPermission); } } } } } else if (roleMembership.EntityType == "Group") { RBACGroup groupDetail = null; if (groupsList != null) { groupDetail = groupsList.Where(g => g.GroupID == roleMembership.EntityID).FirstOrDefault(); } if (groupMembershipsList != null) { // For Group, find all users in the group and repeat the permission output List <RBACGroupMembership> usersInGroups = groupMembershipsList.Where(g => g.GroupID == roleMembership.EntityID).ToList(); if (usersInGroups != null) { foreach (RBACGroupMembership user in usersInGroups) { RBACUser userDetail = null; if (usersList != null) { userDetail = usersList.Where(u => u.UserID == user.UserID).FirstOrDefault(); } List <RBACPermission> permissionsForRoleList = permissionsList.Where(p => p.RoleID == roleMembership.RoleID).ToList(); if (permissionsForRoleList != null) { foreach (RBACPermission permission in permissionsForRoleList) { RBACUserPermission userPermission = new RBACUserPermission(); userPermission.Controller = user.Controller; userPermission.UserName = user.UserName; userPermission.UserID = user.UserID; if (userDetail != null) { userPermission.UserSecurityProvider = userDetail.SecurityProvider; } if (groupDetail != null) { userPermission.GroupName = groupDetail.GroupName; userPermission.GroupSecurityProvider = groupDetail.SecurityProvider; userPermission.GroupID = groupDetail.GroupID; } userPermission.RoleName = permission.RoleName; userPermission.RoleID = permission.RoleID; userPermission.PermissionName = permission.PermissionName; userPermission.PermissionID = permission.PermissionID; userPermission.Allowed = permission.Allowed; userPermission.EntityName = permission.EntityName; userPermission.EntityType = permission.EntityType; userPermission.EntityID = permission.EntityID; userPermissionsList.Add(userPermission); } } } } } } } } userPermissionsList = userPermissionsList.OrderBy(o => o.UserName).ThenBy(o => o.GroupName).ThenBy(o => o.PermissionName).ToList(); FileIOHelper.WriteListToCSVFile(userPermissionsList, new RBACUserPermissionReportMap(), FilePathMap.UserPermissionsIndexFilePath(jobTarget)); #endregion #region Controller Summary loggerConsole.Info("Index Controller Summary"); RBACControllerSummary controller = new RBACControllerSummary(); controller.Controller = jobTarget.Controller; string securityProviderType = FileIOHelper.ReadFileFromPath(FilePathMap.SecurityProviderTypeDataFilePath(jobTarget)); if (securityProviderType != String.Empty) { controller.SecurityProvider = securityProviderType.Replace("\"", ""); } string requireStrongPasswords = FileIOHelper.ReadFileFromPath(FilePathMap.StrongPasswordsDataFilePath(jobTarget)); if (requireStrongPasswords != String.Empty) { bool parsedBool = false; Boolean.TryParse(requireStrongPasswords, out parsedBool); controller.IsStrongPasswords = parsedBool; } if (usersList != null) { controller.NumUsers = usersList.Count; } if (groupsList != null) { controller.NumGroups = groupsList.Count; } if (rolesList != null) { controller.NumRoles = rolesList.Count; } List <RBACControllerSummary> controllerList = new List <RBACControllerSummary>(1); controllerList.Add(controller); FileIOHelper.WriteListToCSVFile(controllerList, new RBACControllerSummaryReportMap(), FilePathMap.RBACControllerSummaryIndexFilePath(jobTarget)); #endregion #region Combine All for Report CSV // If it is the first one, clear out the combined folder if (reportFolderCleaned == false) { FileIOHelper.DeleteFolder(FilePathMap.UsersGroupsRolesPermissionsReportFolderPath()); Thread.Sleep(1000); FileIOHelper.CreateFolder(FilePathMap.UsersGroupsRolesPermissionsReportFolderPath()); reportFolderCleaned = true; } // Append all the individual report files into one if (File.Exists(FilePathMap.UsersIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.UsersIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.UsersReportFilePath(), FilePathMap.UsersIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.GroupsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.GroupsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.GroupsReportFilePath(), FilePathMap.GroupsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.RolesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.RolesIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.RolesReportFilePath(), FilePathMap.RolesIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.PermissionsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.PermissionsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.PermissionsReportFilePath(), FilePathMap.PermissionsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.GroupMembershipsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.GroupMembershipsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.GroupMembershipsReportFilePath(), FilePathMap.GroupMembershipsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.RoleMembershipsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.RoleMembershipsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.RoleMembershipsReportFilePath(), FilePathMap.RoleMembershipsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.UserPermissionsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.UserPermissionsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.UserPermissionsReportFilePath(), FilePathMap.UserPermissionsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.RBACControllerSummaryIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.RBACControllerSummaryIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.RBACControllerSummaryReportFilePath(), FilePathMap.RBACControllerSummaryIndexFilePath(jobTarget)); } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } i++; } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(jobConfiguration) == false) { return(true); } bool reportFolderCleaned = false; // Process each Controller once int i = 0; var controllers = jobConfiguration.Target.GroupBy(t => t.Controller); foreach (var controllerGroup in controllers) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = controllerGroup.ToList()[0]; StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; stepTimingTarget.NumEntities = 0; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Controller Settings loggerConsole.Info("Controller Settings"); List <ControllerSetting> controllerSettingsList = new List <ControllerSetting>(); JArray controllerSettingsArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.ControllerSettingsDataFilePath(jobTarget)); if (controllerSettingsArray != null) { foreach (JObject controllerSettingObject in controllerSettingsArray) { ControllerSetting controllerSetting = new ControllerSetting(); controllerSetting.Controller = jobTarget.Controller; controllerSetting.ControllerLink = String.Format(DEEPLINK_CONTROLLER, controllerSetting.Controller, DEEPLINK_TIMERANGE_LAST_15_MINUTES); controllerSetting.Name = getStringValueFromJToken(controllerSettingObject, "name"); controllerSetting.Description = getStringValueFromJToken(controllerSettingObject, "description"); controllerSetting.Value = getStringValueFromJToken(controllerSettingObject, "value"); controllerSetting.Updateable = getBoolValueFromJToken(controllerSettingObject, "updateable"); controllerSetting.Scope = getStringValueFromJToken(controllerSettingObject, "scope"); controllerSettingsList.Add(controllerSetting); } } controllerSettingsList = controllerSettingsList.OrderBy(c => c.Name).ToList(); FileIOHelper.WriteListToCSVFile(controllerSettingsList, new ControllerSettingReportMap(), FilePathMap.ControllerSettingsIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + controllerSettingsList.Count; #endregion #region HTTP Templates loggerConsole.Info("HTTP Templates"); List <HTTPAlertTemplate> httpTemplatesList = new List <HTTPAlertTemplate>(); JArray httpTemplatesArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.HTTPTemplatesDataFilePath(jobTarget)); JArray httpTemplatesDetailArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.HTTPTemplatesDetailDataFilePath(jobTarget)); if (httpTemplatesArray != null) { foreach (JObject httpTemplateObject in httpTemplatesArray) { HTTPAlertTemplate httpAlertTemplate = new HTTPAlertTemplate(); httpAlertTemplate.Controller = jobTarget.Controller; httpAlertTemplate.Name = getStringValueFromJToken(httpTemplateObject, "name"); httpAlertTemplate.Method = getStringValueFromJToken(httpTemplateObject, "method"); httpAlertTemplate.Scheme = getStringValueFromJToken(httpTemplateObject, "scheme"); httpAlertTemplate.Host = getStringValueFromJToken(httpTemplateObject, "host"); httpAlertTemplate.Port = getIntValueFromJToken(httpTemplateObject, "port"); httpAlertTemplate.Path = getStringValueFromJToken(httpTemplateObject, "path"); httpAlertTemplate.Query = getStringValueFromJToken(httpTemplateObject, "query"); httpAlertTemplate.AuthType = getStringValueFromJToken(httpTemplateObject, "authType"); httpAlertTemplate.AuthUsername = getStringValueFromJToken(httpTemplateObject, "authUsername"); httpAlertTemplate.AuthPassword = getStringValueFromJToken(httpTemplateObject, "authPassword"); httpAlertTemplate.Headers = getStringValueOfObjectFromJToken(httpTemplateObject, "headers", true); if (isTokenPropertyNull(httpTemplateObject, "payloadTemplate") == false) { httpAlertTemplate.ContentType = getStringValueFromJToken(httpTemplateObject["payloadTemplate"], "httpRequestActionMediaType"); httpAlertTemplate.FormData = getStringValueOfObjectFromJToken(httpTemplateObject["payloadTemplate"], "formDataPairs", true); httpAlertTemplate.Payload = getStringValueFromJToken(httpTemplateObject["payloadTemplate"], "payload"); } httpAlertTemplate.ConnectTimeout = getLongValueFromJToken(httpTemplateObject, "connectTimeoutInMillis"); httpAlertTemplate.SocketTimeout = getLongValueFromJToken(httpTemplateObject, "socketTimeoutInMillis"); httpAlertTemplate.ResponseAny = getStringValueOfObjectFromJToken(httpTemplateObject, "responseMatchCriteriaAnyTemplate"); httpAlertTemplate.ResponseNone = getStringValueOfObjectFromJToken(httpTemplateObject, "responseMatchCriteriaNoneTemplate"); if (httpTemplatesDetailArray != null) { JToken httpAlertTemplateToken = httpTemplatesDetailArray.Where(t => t["name"].ToString() == httpAlertTemplate.Name).FirstOrDefault(); if (httpAlertTemplateToken != null) { httpAlertTemplate.TemplateID = getLongValueFromJToken(httpAlertTemplateToken, "id"); } } httpTemplatesList.Add(httpAlertTemplate); } } httpTemplatesList = httpTemplatesList.OrderBy(c => c.Name).ToList(); FileIOHelper.WriteListToCSVFile(httpTemplatesList, new HTTPAlertTemplateReportMap(), FilePathMap.HTTPTemplatesIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + httpTemplatesList.Count; #endregion #region Email Templates loggerConsole.Info("Email Templates"); List <EmailAlertTemplate> emailTemplatesList = new List <EmailAlertTemplate>(); JArray emailTemplatesArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.EmailTemplatesDataFilePath(jobTarget)); JArray emailTemplatesDetailArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.EmailTemplatesDetailDataFilePath(jobTarget)); if (emailTemplatesArray != null) { foreach (JObject emailTemplateObject in emailTemplatesArray) { EmailAlertTemplate emailAlertTemplate = new EmailAlertTemplate(); emailAlertTemplate.Controller = jobTarget.Controller; emailAlertTemplate.Name = getStringValueFromJToken(emailTemplateObject, "name"); emailAlertTemplate.OneEmailPerEvent = getBoolValueFromJToken(emailTemplateObject, "oneEmailPerEvent"); emailAlertTemplate.EventLimit = getLongValueFromJToken(emailTemplateObject, "eventClampLimit"); try { string[] emails = emailTemplateObject["toRecipients"].Select(s => getStringValueFromJToken(s, "value")).ToArray(); emailAlertTemplate.To = String.Join(";", emails); } catch { } try { string[] emails = emailTemplateObject["ccRecipients"].Select(s => getStringValueFromJToken(s, "value")).ToArray(); emailAlertTemplate.CC = String.Join(";", emails); } catch { } try { string[] emails = emailTemplateObject["bccRecipients"].Select(s => getStringValueFromJToken(s, "value")).ToArray(); emailAlertTemplate.BCC = String.Join(";", emails); } catch { } try { string[] emails = emailTemplateObject["testToRecipients"].Select(s => getStringValueFromJToken(s, "value")).ToArray(); emailAlertTemplate.TestTo = String.Join(";", emails); } catch { } try { string[] emails = emailTemplateObject["testCcRecipients"].Select(s => getStringValueFromJToken(s, "value")).ToArray(); emailAlertTemplate.TestCC = String.Join(";", emails); } catch { } try { string[] emails = emailTemplateObject["testBccRecipients"].Select(s => getStringValueFromJToken(s, "value")).ToArray(); emailAlertTemplate.TestBCC = String.Join(";", emails); } catch { } emailAlertTemplate.TestLogLevel = getStringValueFromJToken(emailTemplateObject, "testLogLevel"); emailAlertTemplate.Headers = getStringValueOfObjectFromJToken(emailTemplateObject, "headers", true); emailAlertTemplate.Subject = getStringValueFromJToken(emailTemplateObject, "subject"); emailAlertTemplate.TextBody = getStringValueFromJToken(emailTemplateObject, "textBody"); emailAlertTemplate.HTMLBody = getStringValueFromJToken(emailTemplateObject, "htmlBody"); emailAlertTemplate.IncludeHTMLBody = getBoolValueFromJToken(emailTemplateObject, "includeHtmlBody"); emailAlertTemplate.Properties = getStringValueOfObjectFromJToken(emailTemplateObject, "defaultCustomProperties"); emailAlertTemplate.TestProperties = getStringValueOfObjectFromJToken(emailTemplateObject, "testPropertiesPairs"); emailAlertTemplate.EventTypes = getStringValueOfObjectFromJToken(emailTemplateObject, "eventTypeCountPairs"); if (emailTemplatesDetailArray != null) { JToken emailTemplateDetailToken = emailTemplatesDetailArray.Where(t => t["name"].ToString() == emailAlertTemplate.Name).FirstOrDefault(); if (emailTemplateDetailToken != null) { emailAlertTemplate.TemplateID = getLongValueFromJToken(emailTemplateDetailToken, "id"); } } emailTemplatesList.Add(emailAlertTemplate); } } emailTemplatesList = emailTemplatesList.OrderBy(c => c.Name).ToList(); FileIOHelper.WriteListToCSVFile(emailTemplatesList, new EmailAlertTemplateReportMap(), FilePathMap.EmailTemplatesIndexFilePath(jobTarget)); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + emailTemplatesList.Count; #endregion #region Combine All for Report CSV // If it is the first one, clear out the combined folder if (reportFolderCleaned == false) { FileIOHelper.DeleteFolder(FilePathMap.ControllerSettingsReportFolderPath()); Thread.Sleep(1000); FileIOHelper.CreateFolder(FilePathMap.ControllerSettingsReportFolderPath()); reportFolderCleaned = true; } // Append all the individual report files into one if (File.Exists(FilePathMap.ControllerSettingsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.ControllerSettingsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.ControllerSettingsReportFilePath(), FilePathMap.ControllerSettingsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.HTTPTemplatesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.HTTPTemplatesIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.HTTPTemplatesReportFilePath(), FilePathMap.HTTPTemplatesIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.EmailTemplatesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.EmailTemplatesIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.EmailTemplatesReportFilePath(), FilePathMap.EmailTemplatesIndexFilePath(jobTarget)); } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } i++; } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(jobConfiguration) == false) { return(true); } if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_SIM) == 0) { return(true); } bool reportFolderCleaned = false; // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_SIM) { continue; } StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Tiers List <SIMTier> tiersList = null; JArray tiersRESTList = FileIOHelper.LoadJArrayFromFile(FilePathMap.SIMTiersDataFilePath(jobTarget)); if (tiersRESTList != null) { loggerConsole.Info("Index List of Tiers ({0} entities)", tiersRESTList.Count); tiersList = new List <SIMTier>(tiersRESTList.Count); foreach (JToken tierREST in tiersRESTList) { SIMTier tier = new SIMTier(); tier.ApplicationID = jobTarget.ApplicationID; tier.ApplicationName = jobTarget.Application; tier.Controller = jobTarget.Controller; tier.TierID = (long)tierREST["id"]; tier.TierName = tierREST["name"].ToString(); tier.NumSegments = tier.TierName.Split('|').Length; // Will be filled later tier.NumNodes = 0; updateEntityWithDeeplinks(tier, jobConfiguration.Input.TimeRange); tiersList.Add(tier); } // Sort them tiersList = tiersList.OrderBy(o => o.TierName).ToList(); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + tiersList.Count; } #endregion #region Nodes List <SIMNode> nodesList = null; JArray nodesRESTList = FileIOHelper.LoadJArrayFromFile(FilePathMap.SIMNodesDataFilePath(jobTarget)); if (nodesRESTList != null) { loggerConsole.Info("Index List of Nodes ({0} entities)", nodesRESTList.Count); nodesList = new List <SIMNode>(nodesRESTList.Count); foreach (JToken nodeREST in nodesRESTList) { SIMNode node = new SIMNode(); node.ApplicationID = jobTarget.ApplicationID; node.ApplicationName = jobTarget.Application; node.Controller = jobTarget.Controller; node.NodeID = (long)nodeREST["id"]; node.NodeName = nodeREST["name"].ToString(); if (tiersList != null) { SIMTier tier = tiersList.Where(t => t.TierName == nodeREST["tierName"].ToString()).FirstOrDefault(); if (tier != null) { node.TierID = tier.TierID; node.TierName = tier.TierName; tier.NumNodes++; } } updateEntityWithDeeplinks(node, jobConfiguration.Input.TimeRange); nodesList.Add(node); } // Sort them nodesList = nodesList.OrderBy(o => o.TierName).ThenBy(o => o.NodeName).ToList(); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + nodesList.Count; FileIOHelper.WriteListToCSVFile(tiersList, new SIMTierReportMap(), FilePathMap.SIMTiersIndexFilePath(jobTarget)); FileIOHelper.WriteListToCSVFile(nodesList, new SIMNodeReportMap(), FilePathMap.SIMNodesIndexFilePath(jobTarget)); } #endregion #region Machines List <Machine> machinesList = null; List <MachineProperty> machinePropertiesAndTagsList = null; List <MachineCPU> machineCPUsList = null; List <MachineVolume> machineVolumesList = null; List <MachineNetwork> machineNetworksList = null; List <MachineContainer> machineContainersList = null; List <MachineProcess> machineProcessesList = null; JArray machinesRESTList = FileIOHelper.LoadJArrayFromFile(FilePathMap.SIMMachinesDataFilePath(jobTarget)); if (machinesRESTList != null) { loggerConsole.Info("Index Machines Configuration ({0} entities)", machinesRESTList.Count); machinesList = new List <Machine>(machinesRESTList.Count); machinePropertiesAndTagsList = new List <MachineProperty>(machinesRESTList.Count * 20); machineCPUsList = new List <MachineCPU>(machinesRESTList.Count); machineVolumesList = new List <MachineVolume>(machinesRESTList.Count * 3); machineNetworksList = new List <MachineNetwork>(machinesRESTList.Count * 3); machineContainersList = new List <MachineContainer>(machinesRESTList.Count * 10); machineProcessesList = new List <MachineProcess>(machinesRESTList.Count * 20); int j = 0; foreach (JToken machineRESTinList in machinesRESTList) { JObject machineREST = FileIOHelper.LoadJObjectFromFile(FilePathMap.SIMMachineDataFilePath(jobTarget, machineRESTinList["name"].ToString(), (long)machineRESTinList["id"])); JArray machineContainerREST = FileIOHelper.LoadJArrayFromFile(FilePathMap.SIMMachineDockerContainersDataFilePath(jobTarget, machineRESTinList["name"].ToString(), (long)machineRESTinList["id"])); JArray machineProcessesREST = FileIOHelper.LoadJArrayFromFile(FilePathMap.SIMMachineProcessesDataFilePath(jobTarget, machineRESTinList["name"].ToString(), (long)machineRESTinList["id"], jobConfiguration.Input.TimeRange)); if (machineREST != null) { #region Machine summary Machine machine = new Machine(); machine.ApplicationID = jobTarget.ApplicationID; machine.ApplicationName = jobTarget.Application; machine.Controller = jobTarget.Controller; machine.MachineID = (long)machineREST["id"]; machine.MachineName = machineREST["name"].ToString(); if (nodesList != null) { try { SIMNode nodeSIM = nodesList.Where(t => t.NodeID == (long)machineREST["simNodeId"]).FirstOrDefault(); if (nodeSIM != null) { machine.TierID = nodeSIM.TierID; machine.TierName = nodeSIM.TierName; machine.NodeID = nodeSIM.NodeID; machine.NodeName = nodeSIM.NodeName; } } catch { } } try { machine.MachineType = machineREST["type"].ToString(); } catch { } try { machine.IsHistorical = (bool)machineREST["historical"]; } catch { } try { machine.IsEnabled = (bool)machineREST["simEnabled"]; } catch { } try { machine.DynamicMonitoringMode = machineREST["dynamicMonitoringMode"].ToString(); } catch { } try { machine.HostMachineID = (long)machineREST["agentConfig"]["rawConfig"]["_agentRegistrationSupplementalConfig"]["hostSimMachineId"]; } catch { } try { machine.DotnetCompatibilityMode = (bool)machineREST["agentConfig"]["rawConfig"]["_dotnetRegistrationRequestConfig"]["dotnetCompatibilityMode"]; } catch { } try { machine.ForceMachineInstanceRegistration = (bool)machineREST["agentConfig"]["rawConfig"]["_machineInstanceRegistrationRequestConfig"]["forceMachineInstanceRegistration"]; } catch { } try { machine.AgentConfigFeatures = machineREST["agentConfig"]["rawConfig"]["_features"]["features"].ToString(Newtonsoft.Json.Formatting.None); } catch { } try { machine.ControllerConfigFeatures = machineREST["controllerConfig"]["rawConfig"]["_features"]["features"].ToString(Newtonsoft.Json.Formatting.None); } catch { } if (machineREST["memory"].HasValues == true) { try { machine.MemPhysical = (int)machineREST["memory"]["Physical"]["sizeMb"]; } catch { } try { machine.MemSwap = (int)machineREST["memory"]["Swap"]["sizeMb"]; } catch { } } try { machine.MachineInfo = machineREST["agentConfig"]["rawConfig"]["_agentRegistrationRequestConfig"]["machineInfo"].ToString(); } catch { } try { machine.JVMInfo = machineREST["agentConfig"]["rawConfig"]["_agentRegistrationRequestConfig"]["jvmInfo"].ToString(); } catch { } try { machine.InstallDirectory = machineREST["agentConfig"]["rawConfig"]["_agentRegistrationRequestConfig"]["installDirectory"].ToString(); } catch { } try { machine.AgentVersionRaw = machineREST["agentConfig"]["rawConfig"]["_agentRegistrationRequestConfig"]["agentVersion"].ToString(); } catch { } if (machine.AgentVersionRaw != String.Empty) { // Machine agent looks like that //Machine Agent v4.2.3.2 GA Build Date 2016 - 07 - 11 10:26:01 //Machine Agent v3.7.16.0 GA Build Date 2014 - 02 - 26 21:20:29 //Machine Agent v4.2.3.2 GA Build Date 2016 - 07 - 11 10:17:54 //Machine Agent v4.0.6.0 GA Build Date 2015 - 05 - 11 20:56:44 //Machine Agent v3.8.3.0 GA Build Date 2014 - 06 - 06 17:09:13 //Machine Agent v4.1.7.1 GA Build Date 2015 - 11 - 24 20:49:24 Regex regexVersion = new Regex(@"(?i).*v(\d*\.\d*\.\d*(\.\d*)?).*", RegexOptions.IgnoreCase); Match match = regexVersion.Match(machine.AgentVersionRaw); if (match != null) { if (match.Groups.Count > 1) { machine.AgentVersion = match.Groups[1].Value; if (machine.AgentVersion.Count(v => v == '.') < 3) { machine.AgentVersion = String.Format("{0}.0", machine.AgentVersion); } } } } try { machine.AutoRegisterAgent = (bool)machineREST["agentConfig"]["rawConfig"]["_agentRegistrationRequestConfig"]["autoRegisterAgent"]; } catch { } try { machine.AgentType = machineREST["agentConfig"]["rawConfig"]["_agentRegistrationRequestConfig"]["agentType"].ToString(); } catch { } try { JToken jToken = machineREST["agentConfig"]["rawConfig"]["_agentRegistrationRequestConfig"]["applicationNames"]; if (jToken.Type == JTokenType.Array && jToken.First != null) { if (jToken.Count() > 0) { if (jToken.Count() == 1) { machine.APMApplicationName = jToken.First.ToString(); } else { machine.APMApplicationName = jToken.ToString(Newtonsoft.Json.Formatting.None); } } } } catch { } try { machine.APMTierName = machineREST["agentConfig"]["rawConfig"]["_agentRegistrationRequestConfig"]["tierName"].ToString(); } catch { } try { machine.APMNodeName = machineREST["agentConfig"]["rawConfig"]["_agentRegistrationRequestConfig"]["nodeName"].ToString(); } catch { } #endregion #region Properties if (machineREST["properties"] != null) { foreach (JProperty property in machineREST["properties"]) { MachineProperty machineProp = new MachineProperty(); machineProp.ApplicationID = machine.ApplicationID; machineProp.ApplicationName = machine.ApplicationName; machineProp.Controller = machine.Controller; machineProp.TierID = machine.TierID; machineProp.TierName = machine.TierName; machineProp.NodeID = machine.NodeID; machineProp.NodeName = machine.NodeName; machineProp.MachineID = machine.MachineID; machineProp.MachineName = machine.MachineName; machineProp.PropType = "Property"; machineProp.PropName = property.Name; machineProp.PropValue = property.Value.ToString(); machine.NumProps++; machinePropertiesAndTagsList.Add(machineProp); } } #endregion #region Tags if (machineREST["tags"] != null) { foreach (JProperty property in machineREST["tags"]) { foreach (JToken propertyValue in property.Value) { MachineProperty machineProp = new MachineProperty(); machineProp.ApplicationID = machine.ApplicationID; machineProp.ApplicationName = machine.ApplicationName; machineProp.Controller = machine.Controller; machineProp.TierID = machine.TierID; machineProp.TierName = machine.TierName; machineProp.NodeID = machine.NodeID; machineProp.NodeName = machine.NodeName; machineProp.MachineID = machine.MachineID; machineProp.MachineName = machine.MachineName; machineProp.PropType = "Tag"; machineProp.PropName = property.Name; machineProp.PropValue = propertyValue.ToString(); machine.NumTags++; machinePropertiesAndTagsList.Add(machineProp); } } } #endregion #region CPUs if (machineREST["cpus"] != null) { foreach (JObject cpu in machineREST["cpus"]) { MachineCPU machineCPU = new MachineCPU(); machineCPU.ApplicationID = machine.ApplicationID; machineCPU.ApplicationName = machine.ApplicationName; machineCPU.Controller = machine.Controller; machineCPU.TierID = machine.TierID; machineCPU.TierName = machine.TierName; machineCPU.NodeID = machine.NodeID; machineCPU.NodeName = machine.NodeName; machineCPU.MachineID = machine.MachineID; machineCPU.MachineName = machine.MachineName; machineCPU.CPUID = cpu["cpuId"].ToString(); try { machineCPU.NumCores = (int)cpu["coreCount"]; } catch { } try { machineCPU.NumLogical = (int)cpu["logicalCount"]; } catch { } try { machineCPU.Vendor = cpu["properties"]["Vendor"].ToString(); } catch { } machineCPU.Flags = String.Empty; try { machineCPU.Flags = cpu["properties"]["Flags"].ToString(); } catch { } machineCPU.NumFlags = machineCPU.Flags.Split(' ').Length; try { machineCPU.Model = cpu["properties"]["Model Name"].ToString(); } catch { } try { machineCPU.Speed = cpu["properties"]["Max Speed MHz"].ToString(); } catch { } machine.NumCPUs++; machineCPUsList.Add(machineCPU); } } #endregion #region Volumes if (machineREST["volumes"] != null) { foreach (JObject volume in machineREST["volumes"]) { MachineVolume machineVolume = new MachineVolume(); machineVolume.ApplicationID = machine.ApplicationID; machineVolume.ApplicationName = machine.ApplicationName; machineVolume.Controller = machine.Controller; machineVolume.TierID = machine.TierID; machineVolume.TierName = machine.TierName; machineVolume.NodeID = machine.NodeID; machineVolume.NodeName = machine.NodeName; machineVolume.MachineID = machine.MachineID; machineVolume.MachineName = machine.MachineName; machineVolume.MountPoint = volume["mountPoint"].ToString(); machineVolume.Partition = volume["partition"].ToString(); try { machineVolume.SizeMB = (int)volume["properties"]["Size (MB)"]; } catch { } try { machineVolume.PartitionMetricName = volume["properties"]["PartitionMetricName"].ToString(); } catch { } try { machineVolume.VolumeMetricName = volume["properties"]["VolumeMetricName"].ToString(); } catch { } machine.NumVolumes++; machineVolumesList.Add(machineVolume); } } #endregion #region Networks if (machineREST["networkInterfaces"] != null) { foreach (JObject network in machineREST["networkInterfaces"]) { MachineNetwork machineNetwork = new MachineNetwork(); machineNetwork.ApplicationID = machine.ApplicationID; machineNetwork.ApplicationName = machine.ApplicationName; machineNetwork.Controller = machine.Controller; machineNetwork.TierID = machine.TierID; machineNetwork.TierName = machine.TierName; machineNetwork.NodeID = machine.NodeID; machineNetwork.NodeName = machine.NodeName; machineNetwork.MachineID = machine.MachineID; machineNetwork.MachineName = machine.MachineName; machineNetwork.NetworkName = network["name"].ToString(); machineNetwork.MacAddress = network["macAddress"].ToString(); machineNetwork.IP4Address = network["ip4Address"].ToString(); machineNetwork.IP6Address = network["ip6Address"].ToString(); try { machineNetwork.IP4Gateway = network["properties"]["IPv4 Default Gateway"].ToString(); } catch { } try { machineNetwork.IP6Gateway = network["properties"]["IPv6 Default Gateway"].ToString(); } catch { } try { machineNetwork.PluggedIn = network["properties"]["Plugged In"].ToString(); } catch { } try { machineNetwork.Enabled = network["properties"]["Enabled"].ToString(); } catch { } try { machineNetwork.State = network["properties"]["Operational State"].ToString(); } catch { } try { machineNetwork.Speed = (int)network["properties"]["Speed"]; } catch { } try { machineNetwork.Duplex = network["properties"]["Duplex"].ToString(); } catch { } try { machineNetwork.MTU = network["properties"]["MTU"].ToString(); } catch { } try { machineNetwork.NetworkMetricName = network["properties"]["MetricName"].ToString(); } catch { } machine.NumNetworks++; machineNetworksList.Add(machineNetwork); } } #endregion #region Containers if (machineContainerREST != null) { foreach (JObject container in machineContainerREST) { MachineContainer machineContainer = new MachineContainer(); machineContainer.ApplicationID = machine.ApplicationID; machineContainer.ApplicationName = machine.ApplicationName; machineContainer.Controller = machine.Controller; machineContainer.TierID = machine.TierID; machineContainer.TierName = machine.TierName; machineContainer.NodeID = machine.NodeID; machineContainer.NodeName = machine.NodeName; machineContainer.MachineID = machine.MachineID; machineContainer.MachineName = machine.MachineName; // This worked in 4.4.1, the format is // { // "containerId": "e25bf24eb981bd10d0ef22b78b4b4a73b4cc242813371c43849635c77b3be95f", // "containerName": "settlementServices", // "imageName": "fin-java-services", // "containerSimMachineId": 224 // } try { machineContainer.ContainerID = container["containerId"].ToString(); } catch { } try { machineContainer.ContainerName = container["containerName"].ToString(); } catch { } try { machineContainer.ImageName = container["imageName"].ToString(); } catch { } try { machineContainer.ContainerMachineID = (long)container["containerSimMachineId"]; } catch { } // This worked in 4.4.3.4 // form is identical to what is returned in machine try { machineContainer.ContainerID = container["properties"]["Container|Full Id"].ToString(); } catch { } try { machineContainer.ContainerName = container["properties"]["Container|Name"].ToString(); } catch { } try { machineContainer.ImageName = container["properties"]["Container|Image|Name"].ToString(); } catch { } try { machineContainer.StartedAt = container["properties"]["Container|Started At"].ToString(); } catch { } machine.NumContainers++; machineContainersList.Add(machineContainer); } } #endregion #region Processes if (machineProcessesREST != null) { foreach (JObject machineProcessGroup in machineProcessesREST) { foreach (JObject process in machineProcessGroup["processes"]) { MachineProcess machineProcess = new MachineProcess(); machineProcess.ApplicationID = machine.ApplicationID; machineProcess.ApplicationName = machine.ApplicationName; machineProcess.Controller = machine.Controller; machineProcess.TierID = machine.TierID; machineProcess.TierName = machine.TierName; machineProcess.NodeID = machine.NodeID; machineProcess.NodeName = machine.NodeName; machineProcess.MachineID = machine.MachineID; machineProcess.MachineName = machine.MachineName; machineProcess.Class = process["processClass"].ToString(); machineProcess.ClassID = process["classId"].ToString(); machineProcess.Name = process["name"].ToString(); machineProcess.CommandLine = process["commandLine"].ToString(); try { machineProcess.RealUser = process["properties"]["realUser"].ToString(); } catch { } try { machineProcess.RealGroup = process["properties"]["realGroup"].ToString(); } catch { } try { machineProcess.EffectiveUser = process["effectiveUser"].ToString(); } catch { } try { machineProcess.EffectiveGroup = process["properties"]["effectiveGroup"].ToString(); } catch { } machineProcess.State = process["state"].ToString(); try { machineProcess.NiceLevel = (int)process["properties"]["niceLevel"]; } catch { } machineProcess.PID = (int)process["processId"]; try { machineProcess.ParentPID = (int)process["parentProcessId"]; } catch { } try { machineProcess.PGID = (int)process["properties"]["pgid"]; } catch { } try { long timeStamp = (long)process["startTime"]; machineProcess.StartTime = UnixTimeHelper.ConvertFromUnixTimestamp(timeStamp); } catch { } try { long timeStamp = (long)process["endTime"]; machineProcess.EndTime = UnixTimeHelper.ConvertFromUnixTimestamp(timeStamp); } catch { } machineProcessesList.Add(machineProcess); } } } #endregion updateEntityWithDeeplinks(machine, jobConfiguration.Input.TimeRange); machinesList.Add(machine); } j++; if (j % 50 == 0) { Console.Write("[{0}].", j); } } // Sort them machinesList = machinesList.OrderBy(o => o.TierName).ThenBy(o => o.MachineName).ToList(); machinePropertiesAndTagsList = machinePropertiesAndTagsList.OrderBy(o => o.TierName).ThenBy(o => o.MachineName).ThenBy(o => o.PropType).ThenBy(o => o.PropName).ToList(); machineCPUsList = machineCPUsList.OrderBy(o => o.TierName).ThenBy(o => o.MachineName).ToList(); machineVolumesList = machineVolumesList.OrderBy(o => o.TierName).ThenBy(o => o.MachineName).ThenBy(o => o.MountPoint).ToList(); machineNetworksList = machineNetworksList.OrderBy(o => o.TierName).ThenBy(o => o.MachineName).ThenBy(o => o.NetworkName).ToList(); machineContainersList = machineContainersList.OrderBy(o => o.TierName).ThenBy(o => o.MachineName).ThenBy(o => o.ContainerName).ToList(); machineProcessesList = machineProcessesList.OrderBy(o => o.TierName).ThenBy(o => o.MachineName).ThenBy(o => o.Class).ToList(); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + machinesList.Count; FileIOHelper.WriteListToCSVFile(machinesList, new MachineReportMap(), FilePathMap.SIMMachinesIndexFilePath(jobTarget)); FileIOHelper.WriteListToCSVFile(machinePropertiesAndTagsList, new MachinePropertyReportMap(), FilePathMap.SIMMachinePropertiesIndexFilePath(jobTarget)); FileIOHelper.WriteListToCSVFile(machineCPUsList, new MachineCPUReportMap(), FilePathMap.SIMMachineCPUsIndexFilePath(jobTarget)); FileIOHelper.WriteListToCSVFile(machineVolumesList, new MachineVolumeReportMap(), FilePathMap.SIMMachineVolumesIndexFilePath(jobTarget)); FileIOHelper.WriteListToCSVFile(machineNetworksList, new MachineNetworkReportMap(), FilePathMap.SIMMachineNetworksIndexFilePath(jobTarget)); FileIOHelper.WriteListToCSVFile(machineContainersList, new MachineContainerReportMap(), FilePathMap.SIMMachineContainersIndexFilePath(jobTarget)); FileIOHelper.WriteListToCSVFile(machineProcessesList, new MachineProcessReportMap(), FilePathMap.SIMMachineProcessesIndexFilePath(jobTarget)); Console.WriteLine("{0} done", machinesRESTList.Count); } #endregion #region Application loggerConsole.Info("Index Application"); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + 1; List <SIMApplication> applicationsList = new List <SIMApplication>(1); SIMApplication applicationRow = new SIMApplication(); applicationRow.ApplicationID = jobTarget.ApplicationID; applicationRow.ApplicationName = jobTarget.Application; applicationRow.Controller = jobTarget.Controller; if (tiersList != null) { applicationRow.NumTiers = tiersList.Count; } if (nodesList != null) { applicationRow.NumNodes = nodesList.Count; } if (machinesList != null) { applicationRow.NumMachines = machinesList.Count; } updateEntityWithDeeplinks(applicationRow, jobConfiguration.Input.TimeRange); applicationsList.Add(applicationRow); FileIOHelper.WriteListToCSVFile(applicationsList, new SIMApplicationReportMap(), FilePathMap.SIMApplicationIndexFilePath(jobTarget)); #endregion #region Combine All for Report CSV // If it is the first one, clear out the combined folder if (reportFolderCleaned == false) { FileIOHelper.DeleteFolder(FilePathMap.SIMEntitiesReportFolderPath()); Thread.Sleep(1000); FileIOHelper.CreateFolder(FilePathMap.SIMEntitiesReportFolderPath()); reportFolderCleaned = true; } // Append all the individual application files into one if (File.Exists(FilePathMap.SIMApplicationIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.SIMApplicationIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.SIMApplicationsReportFilePath(), FilePathMap.SIMApplicationIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.SIMTiersIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.SIMTiersIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.SIMTiersReportFilePath(), FilePathMap.SIMTiersIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.SIMNodesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.SIMNodesIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.SIMNodesReportFilePath(), FilePathMap.SIMNodesIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.SIMMachinesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.SIMMachinesIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.SIMMachinesReportFilePath(), FilePathMap.SIMMachinesIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.SIMMachinePropertiesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.SIMMachinePropertiesIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.SIMMachinePropertiesReportFilePath(), FilePathMap.SIMMachinePropertiesIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.SIMMachineCPUsIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.SIMMachineCPUsIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.SIMMachineCPUsReportFilePath(), FilePathMap.SIMMachineCPUsIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.SIMMachineVolumesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.SIMMachineVolumesIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.SIMMachineVolumesReportFilePath(), FilePathMap.SIMMachineVolumesIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.SIMMachineNetworksIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.SIMMachineNetworksIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.SIMMachineNetworksReportFilePath(), FilePathMap.SIMMachineNetworksIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.SIMMachineContainersIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.SIMMachineContainersIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.SIMMachineContainersReportFilePath(), FilePathMap.SIMMachineContainersIndexFilePath(jobTarget)); } if (File.Exists(FilePathMap.SIMMachineProcessesIndexFilePath(jobTarget)) == true && new FileInfo(FilePathMap.SIMMachineProcessesIndexFilePath(jobTarget)).Length > 0) { FileIOHelper.AppendTwoCSVFiles(FilePathMap.SIMMachineProcessesReportFilePath(), FilePathMap.SIMMachineProcessesIndexFilePath(jobTarget)); } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }