public static void ExecuteJobThroughSteps(ProgramOptions programOptions) { // Read job file from the location JobConfiguration jobConfiguration = FileIOHelper.ReadJobConfigurationFromFile(programOptions.OutputJobFilePath); if (jobConfiguration == null) { loggerConsole.Error("Unable to load job input file {0}", programOptions.InputJobFilePath); return; } #region Output diagnostic parameters to log loggerConsole.Info("Starting job from status {0}({0:d})", jobConfiguration.Status); logger.Info("Starting job from status {0}({0:d})", jobConfiguration.Status); logger.Info("Job input: TimeRange.From='{0:o}', TimeRange.To='{1:o}', Time ranges='{2}', Flowmaps='{3}', Metrics='{4}', Snapshots='{5}', Configuration='{6}', Events='{7}'", jobConfiguration.Input.TimeRange.From, jobConfiguration.Input.TimeRange.To, jobConfiguration.Input.HourlyTimeRanges.Count, jobConfiguration.Input.Flowmaps, jobConfiguration.Input.Metrics, jobConfiguration.Input.Snapshots, jobConfiguration.Input.Configuration, jobConfiguration.Input.Events); if (jobConfiguration.Input.MetricsSelectionCriteria != null) { logger.Info("Job input: MetricsSelectionCriteria='{0}'", String.Join(",", jobConfiguration.Input.MetricsSelectionCriteria)); } if (jobConfiguration.Input.SnapshotSelectionCriteria != null) { PropertyInfo[] pis = jobConfiguration.Input.SnapshotSelectionCriteria.TierType.GetType().GetProperties(); StringBuilder sb = new StringBuilder(16 * pis.Length); foreach (PropertyInfo pi in pis) { sb.AppendFormat("{0}={1}, ", pi.Name, pi.GetValue(jobConfiguration.Input.SnapshotSelectionCriteria.TierType)); } logger.Info("Job input, SnapshotSelectionCriteria: Tiers='{0}', TierTypes='{1}'", String.Join(",", jobConfiguration.Input.SnapshotSelectionCriteria.Tiers), sb.ToString()); pis = jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactionType.GetType().GetProperties(); sb = new StringBuilder(16 * pis.Length); foreach (PropertyInfo pi in pis) { sb.AppendFormat("{0}={1}, ", pi.Name, pi.GetValue(jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactionType)); } logger.Info("Job input, SnapshotSelectionCriteria: BusinessTransactions='{0}', BusinessTransactionType='{1}'", String.Join(",", jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactions), sb.ToString()); logger.Info("Job input, SnapshotSelectionCriteria: UserExperience.Normal='{0}', UserExperience.Slow='{1}', UserExperience.VerySlow='{2}', UserExperience.Stall='{3}', UserExperience.Error='{4}'", jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Normal, jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Slow, jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.VerySlow, jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Stall, jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Error); logger.Info("Job input, SnapshotSelectionCriteria: SnapshotType.Full='{0}', SnapshotType.Partial='{1}', SnapshotType.None='{2}'", jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType.Full, jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType.Partial, jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType.None); } if (jobConfiguration.Input.ConfigurationComparisonReferenceCriteria != null) { logger.Info("Job input: ConfigurationComparisonReferenceCriteria.Controller='{0}', ConfigurationComparisonReferenceCriteria.Application='{1}'", jobConfiguration.Input.ConfigurationComparisonReferenceCriteria.Controller, jobConfiguration.Input.ConfigurationComparisonReferenceCriteria.Application); } logger.Info("Job output: DetectedEntities='{0}', EntityMetrics='{1}', EntityDetails='{2}', Snapshots='{3}', Configuration='{4}', Events='{5}'", jobConfiguration.Output.DetectedEntities, jobConfiguration.Output.EntityMetrics, jobConfiguration.Output.EntityDetails, jobConfiguration.Output.Snapshots, jobConfiguration.Output.Configuration, jobConfiguration.Output.Events); foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges) { logger.Info("Expanded time ranges: From='{0:o}', To='{1:o}'", jobTimeRange.From, jobTimeRange.To); } #endregion // Run the step and move to next until things are done while (jobConfiguration.Status != JobStatus.Done && jobConfiguration.Status != JobStatus.Error) { loggerConsole.Info("Executing job step {0}({0:d})", jobConfiguration.Status); logger.Info("Executing job step {0}({0:d})", jobConfiguration.Status); JobStepBase jobStep = getJobStepFromFactory(jobConfiguration.Status); if (jobStep != null) { if (jobStep.Execute(programOptions, jobConfiguration) == false) { jobConfiguration.Status = JobStatus.Error; } } if (jobConfiguration.Status != JobStatus.Error) { jobConfiguration.Status = jobStepsLinked.Find(jobConfiguration.Status).Next.Value; } // Save the resulting JSON file to the job target folder if (FileIOHelper.WriteJobConfigurationToFile(jobConfiguration, programOptions.OutputJobFilePath) == false) { loggerConsole.Error("Unable to write job input file {0}", programOptions.OutputJobFilePath); return; } } }
public static void Main(string[] args) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); try { logger.Trace("Starting at local {0:o}/UTC {1:o}, Version={2}, Parameters={3}", DateTime.Now, DateTime.UtcNow, Assembly.GetEntryAssembly().GetName().Version, String.Join(" ", args)); logger.Trace("Timezone {0} {1} {2}", TimeZoneInfo.Local.DisplayName, TimeZoneInfo.Local.StandardName, TimeZoneInfo.Local.BaseUtcOffset); logger.Trace("Framework {0}", RuntimeInformation.FrameworkDescription); logger.Trace("OS Architecture {0}", RuntimeInformation.OSArchitecture); logger.Trace("OS {0}", RuntimeInformation.OSDescription); logger.Trace("Process Architecture {0}", RuntimeInformation.ProcessArchitecture); #region Parse input parameters // Parse parameters ProgramOptions programOptions = new ProgramOptions(); if (Parser.Default.ParseArguments(args, programOptions) == false) { logger.Error("Could not parse command line arguments into ProgramOptions"); loggerConsole.Error("Could not parse command line arguments into ProgramOptions"); return; } #endregion #region Validate job file exists programOptions.InputJobFilePath = Path.GetFullPath(programOptions.InputJobFilePath); logger.Info("Checking input job file {0}", programOptions.InputJobFilePath); loggerConsole.Info("Checking input job file {0}", programOptions.InputJobFilePath); if (File.Exists(programOptions.InputJobFilePath) == false) { logger.Error("Job file {0} does not exist", programOptions.InputJobFilePath); loggerConsole.Error("Job file {0} does not exist", programOptions.InputJobFilePath); return; } #endregion #region Create output folder // If output folder isn't specified, assume output folder to be: // Windows: at the root of C: on Windows // Mac/Linux: a child of %HOME% path if (programOptions.OutputFolderPath == null || programOptions.OutputFolderPath.Length == 0) { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) == true) { programOptions.OutputFolderPath = @"C:\AppD.Dexter.Out"; } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) == true || RuntimeInformation.IsOSPlatform(OSPlatform.OSX) == true) { programOptions.OutputFolderPath = Path.Combine(Environment.GetEnvironmentVariable("HOME"), "AppD.Dexter.Out"); } } programOptions.OutputFolderPath = Path.GetFullPath(programOptions.OutputFolderPath); logger.Info("Creating output folder {0}", programOptions.OutputFolderPath); loggerConsole.Info("Checking output folder {0}", programOptions.OutputFolderPath); if (FileIOHelper.CreateFolder(programOptions.OutputFolderPath) == false) { logger.Error("Unable to create output folder={0}", programOptions.OutputFolderPath); loggerConsole.Error("Unable to create output folder={0}", programOptions.OutputFolderPath); return; } #endregion #region Create job output folder // Set up the output job folder and job file path programOptions.JobName = Path.GetFileNameWithoutExtension(programOptions.InputJobFilePath); programOptions.OutputJobFolderPath = Path.Combine(programOptions.OutputFolderPath, programOptions.JobName); programOptions.OutputJobFilePath = Path.Combine(programOptions.OutputJobFolderPath, "jobparameters.json"); programOptions.ProgramLocationFolderPath = AppDomain.CurrentDomain.BaseDirectory; // Create job folder if it doesn't exist // or // Clear out job folder if it already exists and restart of the job was requested if (programOptions.RestartJobFromBeginning) { if (FileIOHelper.DeleteFolder(programOptions.OutputJobFolderPath) == false) { logger.Error("Unable to clear job folder {0}", programOptions.OutputJobFolderPath); loggerConsole.Error("Unable to clear job folder {0}", programOptions.OutputJobFolderPath); return; } // Sleep after deleting to let the file system catch up Thread.Sleep(1000); } logger.Info("Creating job output folder {0}", programOptions.OutputJobFolderPath); loggerConsole.Info("Creating job output folder {0}", programOptions.OutputJobFolderPath); if (FileIOHelper.CreateFolder(programOptions.OutputJobFolderPath) == false) { logger.Error("Unable to create job output folder={0}", programOptions.OutputJobFolderPath); loggerConsole.Error("Unable to create job output folder={0}", programOptions.OutputJobFolderPath); return; } #endregion #region Process input job file to output job file // Check if this job file was already already validated and exists in target folder loggerConsole.Info("Processing input job file to output job file"); if (Directory.Exists(programOptions.OutputJobFolderPath) == false || File.Exists(programOptions.OutputJobFilePath) == false) { // New job // Validate job file for validity if the job is new // Expand list of targets from the input file // Save validated job file to the output directory // Load job configuration JobConfiguration jobConfiguration = FileIOHelper.ReadJobConfigurationFromFile(programOptions.InputJobFilePath); if (jobConfiguration == null) { loggerConsole.Error("Unable to load job input file {0}", programOptions.InputJobFilePath); return; } #region Validate Input if (jobConfiguration.Input == null) { logger.Error("Job File Problem: Input can not be empty"); loggerConsole.Error("Job File Problem: Input can not be empty"); return; } // Validate input time range selection if (jobConfiguration.Input.TimeRange == null || jobConfiguration.Input.TimeRange.From == null || jobConfiguration.Input.TimeRange.From == DateTime.MinValue) { logger.Error("Job File Problem: Input.TimeRange.From can not be empty"); loggerConsole.Error("Job File Problem: Input.TimeRange.From can not be empty"); return; } else if (jobConfiguration.Input.TimeRange == null || jobConfiguration.Input.TimeRange.To == null || jobConfiguration.Input.TimeRange.To == DateTime.MinValue) { logger.Error("Job File Problem: Input.TimeRange.To can not be empty"); loggerConsole.Error("Job File Problem: Input.TimeRange.To can not be empty"); return; } jobConfiguration.Input.TimeRange.From = jobConfiguration.Input.TimeRange.From.ToUniversalTime(); jobConfiguration.Input.TimeRange.To = jobConfiguration.Input.TimeRange.To.ToUniversalTime(); if (jobConfiguration.Input.TimeRange.From > jobConfiguration.Input.TimeRange.To) { logger.Error("Job File Problem: Input.TimeRange.From='{0:o}' can not be > Input.TimeRange.To='{1:o}'", jobConfiguration.Input.TimeRange.From, jobConfiguration.Input.TimeRange.To); loggerConsole.Error("Job File Problem: Input.TimeRange.From='{0:o}' can not be > Input.TimeRange.To='{1:o}'", jobConfiguration.Input.TimeRange.From, jobConfiguration.Input.TimeRange.To); return; } else if (jobConfiguration.Input.TimeRange.From > DateTime.UtcNow) { logger.Error("Job File Problem: Input.TimeRange.From='{0:o}' can not be in the future", jobConfiguration.Input.TimeRange.From); loggerConsole.Error("Job File Problem: Input.TimeRange.From='{0:o}' can not be in the future", jobConfiguration.Input.TimeRange.From); return; } // Validate Metrics selection if (jobConfiguration.Input.MetricsSelectionCriteria == null) { jobConfiguration.Input.MetricsSelectionCriteria = new string[0]; } // Validate Snapshot selection if (jobConfiguration.Input.SnapshotSelectionCriteria == null) { jobConfiguration.Input.SnapshotSelectionCriteria = new JobSnapshotSelectionCriteria(); } if (jobConfiguration.Input.SnapshotSelectionCriteria.Tiers == null) { jobConfiguration.Input.SnapshotSelectionCriteria.Tiers = new string[0]; } if (jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactions == null) { jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactions = new string[0]; } if (jobConfiguration.Input.SnapshotSelectionCriteria.TierType == null) { jobConfiguration.Input.SnapshotSelectionCriteria.TierType = new JobTierType(); jobConfiguration.Input.SnapshotSelectionCriteria.TierType.All = true; } if (jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactionType == null) { jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactionType = new JobBusinessTransactionType(); jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactionType.All = true; } if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience == null) { jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience = new JobUserExperience(); jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Normal = true; jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Slow = true; jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.VerySlow = true; jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Stall = true; jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Error = true; } if (jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType == null) { jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType = new JobSnapshotType(); jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType.Full = true; jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType.Partial = true; jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType.None = true; } // Validate Configuration Comparison selection if (jobConfiguration.Input.ConfigurationComparisonReferenceCriteria == null) { jobConfiguration.Input.ConfigurationComparisonReferenceCriteria = new JobTarget(); jobConfiguration.Input.ConfigurationComparisonReferenceCriteria.Controller = JobStepBase.BLANK_APPLICATION_CONTROLLER; jobConfiguration.Input.ConfigurationComparisonReferenceCriteria.Controller = JobStepBase.BLANK_APPLICATION_APPLICATION; } #endregion #region Validate Output if (jobConfiguration.Output == null) { logger.Error("Job File Problem: Output can not be empty"); loggerConsole.Error("Job File Problem: Output can not be empty"); return; } #endregion #region Expand time ranges into hourly chunks // Prepare list of time ranges that goes from the Hour:00 of the From to the Hour:59 of the To jobConfiguration.Input.HourlyTimeRanges = new List <JobTimeRange>(); DateTime intervalStartTime = jobConfiguration.Input.TimeRange.From; DateTime intervalEndTime = new DateTime( intervalStartTime.Year, intervalStartTime.Month, intervalStartTime.Day, intervalStartTime.Hour, 0, 0, DateTimeKind.Utc).AddHours(1); do { TimeSpan timeSpan = intervalEndTime - jobConfiguration.Input.TimeRange.To; if (timeSpan.TotalMinutes >= 0) { jobConfiguration.Input.HourlyTimeRanges.Add(new JobTimeRange { From = intervalStartTime, To = jobConfiguration.Input.TimeRange.To }); break; } else { jobConfiguration.Input.HourlyTimeRanges.Add(new JobTimeRange { From = intervalStartTime, To = intervalEndTime }); } intervalStartTime = intervalEndTime; intervalEndTime = intervalStartTime.AddHours(1); }while (true); #endregion #region Validate list of targets // Validate list of targets if (jobConfiguration.Target == null || jobConfiguration.Target.Count == 0) { logger.Error("Job File Problem: No targets to work on"); loggerConsole.Error("Job File Problem: No targets to work on"); return; } #endregion #region Expand list of targets // Process each target and validate the controller authentication, as well as create multiple per-application entries if there is a regex match List <JobTarget> expandedJobTargets = new List <JobTarget>(jobConfiguration.Target.Count); for (int i = 0; i < jobConfiguration.Target.Count; i++) { JobTarget jobTarget = jobConfiguration.Target[i]; jobTarget.ApplicationID = -1; #region Validate target Controller properties against being empty bool isTargetValid = true; if (jobTarget.Controller == null || jobTarget.Controller == string.Empty) { logger.Warn("Target {0} property {1} is empty", i + 1, "Controller"); loggerConsole.Warn("Target {0} property {1} is empty", i + 1, "Controller"); isTargetValid = false; } if (jobTarget.UserName == null || jobTarget.UserName == string.Empty) { logger.Warn("Target {0} property {1} is empty", i + 1, "UserName"); loggerConsole.Warn("Target {0} property {1} is empty", i + 1, "UserName"); isTargetValid = false; } if (jobTarget.Application == null || jobTarget.Application == string.Empty) { logger.Warn("Target {0} property {1} is empty", i + 1, "Application"); loggerConsole.Warn("Target {0} property {1} is empty", i + 1, "Application"); isTargetValid = false; } if (isTargetValid == false) { jobTarget.Status = JobTargetStatus.InvalidConfiguration; expandedJobTargets.Add(jobTarget); continue; } #endregion #region Get credential or prompt for it if (jobTarget.UserPassword == null || jobTarget.UserPassword == string.Empty) { logger.Warn("Target {0} property {1} is empty", i + 1, "UserPassword"); loggerConsole.Warn("Target {0} property {1} is empty", i + 1, "UserPassword"); loggerConsole.Warn("Enter Password for user {0} for {1}:", jobTarget.UserName, jobTarget.Controller); String password = ReadPassword('*'); Console.WriteLine(); if (password.Length == 0) { logger.Warn("User specified empty password"); loggerConsole.Warn("Password can not be empty"); jobTarget.Status = JobTargetStatus.NoController; continue; } jobTarget.UserPassword = AESEncryptionHelper.Encrypt(password); } #endregion #region Validate target Controller is accessible // If reached here, we have all the properties to go query for list of Applications //ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, jobTarget.UserPassword); ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)); if (controllerApi.IsControllerAccessible() == false) { logger.Warn("Target [{0}] Controller {1} not accessible", i + 1, controllerApi); loggerConsole.Warn("Target [{0}] Controller {1} not accessible", i + 1, controllerApi); jobTarget.Status = JobTargetStatus.NoController; jobTarget.UserPassword = AESEncryptionHelper.Encrypt(jobTarget.UserPassword); expandedJobTargets.Add(jobTarget); continue; } #endregion #region Expand list of Applications using regex, if present // Now we know we have access to Controller. Let's get Applications and expand them into multiple targets if there is a wildcard/regex string applicationsJSON = controllerApi.GetListOfApplications(); JArray applicationsInTarget = JArray.Parse(applicationsJSON); IEnumerable <JToken> applicationsMatchingCriteria = null; if (jobTarget.NameRegex == true) { if (jobTarget.Application == "*") { jobTarget.Application = ".*"; } Regex regexApplication = new Regex(jobTarget.Application, RegexOptions.IgnoreCase); applicationsMatchingCriteria = applicationsInTarget.Where( app => regexApplication.Match(app["name"].ToString()).Success == true); } else { applicationsMatchingCriteria = applicationsInTarget.Where( app => String.Compare(app["name"].ToString(), jobTarget.Application, true) == 0); } if (applicationsMatchingCriteria.Count() == 0) { logger.Warn("Target [{0}] Controller {1} does not have Application {2}", i + 1, jobTarget.Controller, jobTarget.Application); loggerConsole.Warn("Target [{0}] Controller {1} does not have Application {2}", i + 1, jobTarget.Controller, jobTarget.Application); jobTarget.Status = JobTargetStatus.NoApplication; expandedJobTargets.Add(jobTarget); continue; } foreach (JObject application in applicationsMatchingCriteria) { // Create a copy of target application for each individual application JobTarget jobTargetExpanded = new JobTarget(); jobTargetExpanded.Controller = jobTarget.Controller.TrimEnd('/'); jobTargetExpanded.UserName = jobTarget.UserName; jobTargetExpanded.UserPassword = AESEncryptionHelper.Encrypt(AESEncryptionHelper.Decrypt(jobTarget.UserPassword)); jobTargetExpanded.Application = application["name"].ToString(); jobTargetExpanded.ApplicationID = (int)application["id"]; // Add status to each individual application jobTargetExpanded.Status = JobTargetStatus.ConfigurationValid; expandedJobTargets.Add(jobTargetExpanded); logger.Info("Target [{0}] Controller {1} Application {2}=>{3}", i + 1, jobTarget.Controller, jobTarget.Application, jobTargetExpanded.Application); loggerConsole.Info("Target [{0}] Controller {1} Application {2}=>{3}", i + 1, jobTarget.Controller, jobTarget.Application, jobTargetExpanded.Application); } #endregion } // Final check for fat-fingered passwords or no internet access if (expandedJobTargets.Count(t => t.Status == JobTargetStatus.ConfigurationValid) == 0) { logger.Error("Job File Problem: Expanded targets but not a single valid target to work on"); loggerConsole.Error("Job File Problem: Expanded targets but not a single valid target to work on"); return; } // Sort them to be pretty expandedJobTargets = expandedJobTargets.OrderBy(o => o.Controller).ThenBy(o => o.Application).ToList(); // Save expanded targets jobConfiguration.Target = expandedJobTargets; #endregion // Add status to the overall job jobConfiguration.Status = JobStatus.ExtractControllerApplicationsAndEntities; // Save the resulting JSON file to the job target folder if (FileIOHelper.WriteJobConfigurationToFile(jobConfiguration, programOptions.OutputJobFilePath) == false) { loggerConsole.Error("Unable to write job input file {0}", programOptions.OutputJobFilePath); return; } } #endregion logger.Trace("Executing ProgramOptions:\r\n{0}", programOptions); loggerConsole.Trace("Executing ProgramOptions:\r\n{0}", programOptions); // Go to work on the expanded and validated job file JobStepRouter.ExecuteJobThroughSteps(programOptions); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); } finally { stopWatch.Stop(); logger.Info("Application execution took {0:c} ({1} ms)", stopWatch.Elapsed, stopWatch.ElapsedMilliseconds); loggerConsole.Trace("Application execution took {0:c} ({1} ms)", stopWatch.Elapsed, stopWatch.ElapsedMilliseconds); // Flush all the logs before shutting down LogManager.Flush(); //LogManager.Configuration.AllTargets // .OfType<BufferingTargetWrapper>() // .ToList() // .ForEach(b => b.Flush(e => // { // do nothing here // })); } }