public int Execute(string[] args) { StartTime = DateTime.Now; try { if (!Config.PopulateConfig(args)) { Config.SaveConfigFile(); return(1); } Log.Info($"version: {Version}"); ParallelConfig = new ParallelOptions { MaxDegreeOfParallelism = Config.Threads }; ServicePointManager.DefaultConnectionLimit = Config.Threads * MaxThreadMultiplier; ThreadPool.SetMinThreads(Config.Threads * MinThreadMultiplier, Config.Threads * MinThreadMultiplier); ThreadPool.SetMaxThreads(Config.Threads * MaxThreadMultiplier, Config.Threads * MaxThreadMultiplier); if (!InitializeKusto() | !InitializeLogAnalytics()) { return(1); } if (Config.SasEndpointInfo.IsPopulated()) { DownloadAzureData(); } else if (Config.IsCacheLocationPreConfigured()) { UploadCacheData(); } CustomTaskManager.WaitAll(); FinalizeKusto(); CustomTaskManager.Close(); if (Config.DeleteCache & Config.IsCacheLocationPreConfigured()) { Log.Info($"Deleting outputlocation: {Config.CacheLocation}"); try { Directory.Delete($"{Config.CacheLocation}", true); } catch (Exception ex) { Log.Exception($"{ex}"); } } Config.DisplayStatus(); Config.SaveConfigFile(); Log.Info($"{TotalFilesEnumerated} files enumerated."); Log.Info($"{TotalFilesMatched} files matched."); Log.Info($"{TotalFilesDownloaded} files downloaded."); Log.Info($"{TotalFilesSkipped} files skipped."); Log.Info($"{TotalFilesFormatted} files formatted."); Log.Info($"{TotalErrors} errors."); Log.Info($"{TotalRecords} records."); Log.Info($"total execution time in minutes: { (DateTime.Now - StartTime).TotalMinutes.ToString("F2") }"); return(0); } catch (Exception ex) { Log.Exception($"{ex}"); return(1); } finally { Log.Close(); } }
public int Collect(string[] args, ConfigurationOptions options) { try { if (!Config.PopulateConfig(args, options)) { Config.SaveConfigFile(); return(1); } Log.Info($"version: {Version}"); _parallelConfig = new ParallelOptions { MaxDegreeOfParallelism = Config.Threads }; ServicePointManager.DefaultConnectionLimit = Config.Threads * MaxThreadMultiplier; ServicePointManager.Expect100Continue = true; ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12; ThreadPool.SetMinThreads(Config.Threads * MinThreadMultiplier, Config.Threads * MinThreadMultiplier); ThreadPool.SetMaxThreads(Config.Threads * MaxThreadMultiplier, Config.Threads * MaxThreadMultiplier); if (Config.NoProgressTimeoutMin > 0) { _noProgressTimer = new Timer(NoProgressCallback, null, 0, 60 * 1000); } if (!InitializeKusto() | !InitializeLogAnalytics()) { return(1); } if (Config.SasEndpointInfo.IsPopulated()) { DownloadAzureData(); } else if (Config.IsCacheLocationPreConfigured()) { UploadCacheData(); } CustomTaskManager.WaitAll(); FinalizeKusto(); CustomTaskManager.Close(); if (Config.DeleteCache & Config.IsCacheLocationPreConfigured()) { Log.Info($"Deleting outputlocation: {Config.CacheLocation}"); try { Directory.Delete($"{Config.CacheLocation}", true); } catch (Exception ex) { Log.Exception($"{ex}"); } } Config.DisplayStatus(); Config.SaveConfigFile(); TotalErrors += Log.LogErrors; Log.Last($"{TotalFilesEnumerated} files enumerated."); Log.Last($"{TotalFilesMatched} files matched."); Log.Last($"{TotalFilesDownloaded} files downloaded."); Log.Last($"{TotalFilesSkipped} files skipped."); Log.Last($"{TotalFilesFormatted} files formatted."); Log.Last($"{TotalErrors} errors."); Log.Last($"{TotalRecords} records."); if (TotalFilesEnumerated > 0) { if (Config.FileType != FileTypesEnum.table) { DateTime discoveredMinDateTime = new DateTime(DiscoveredMinDateTicks); DateTime discoveredMaxDateTime = new DateTime(DiscoveredMaxDateTicks); Log.Last($"discovered time range: {discoveredMinDateTime.ToString("o")} - {discoveredMaxDateTime.ToString("o")}", ConsoleColor.Green); if (discoveredMinDateTime.Ticks > Config.EndTimeUtc.Ticks | discoveredMaxDateTime.Ticks < Config.StartTimeUtc.Ticks) { Log.Last($"error: configured time range not within discovered time range. configured time range: {Config.StartTimeUtc} - {Config.EndTimeUtc}", ConsoleColor.Red); } } if (TotalFilesMatched + TotalRecords == 0 && (!string.IsNullOrEmpty(Config.UriFilter) | !string.IsNullOrEmpty(Config.ContainerFilter) | !string.IsNullOrEmpty(Config.NodeFilter))) { Log.Last("0 records found and filters are configured. verify filters and / or try time range are correct.", ConsoleColor.Yellow); } else if (TotalFilesMatched + TotalRecords == 0) { Log.Last("0 records found. verify time range is correct.", ConsoleColor.Yellow); } } else { Log.Last("0 files enumerated.", ConsoleColor.Red); } Log.Last($"total execution time in minutes: { (DateTime.Now - StartTime).TotalMinutes.ToString("F2") }"); return(TotalErrors); } catch (Exception ex) { Log.Exception($"{ex}"); return(1); } finally { Log.Close(); } }