public int Execute(string[] args)
{
StartTime = DateTime.Now;
try
{
if (!Config.PopulateConfig(args))
{
Config.SaveConfigFile();
return(1);
}
Log.Info($"version: {Version}");
ParallelConfig = new ParallelOptions {
MaxDegreeOfParallelism = Config.Threads
};
ServicePointManager.DefaultConnectionLimit = Config.Threads * MaxThreadMultiplier;
ThreadPool.SetMinThreads(Config.Threads * MinThreadMultiplier, Config.Threads * MinThreadMultiplier);
ThreadPool.SetMaxThreads(Config.Threads * MaxThreadMultiplier, Config.Threads * MaxThreadMultiplier);
if (!InitializeKusto() | !InitializeLogAnalytics())
{
return(1);
}
if (Config.SasEndpointInfo.IsPopulated())
{
DownloadAzureData();
}
else if (Config.IsCacheLocationPreConfigured())
{
UploadCacheData();
}
CustomTaskManager.WaitAll();
FinalizeKusto();
CustomTaskManager.Close();
if (Config.DeleteCache & Config.IsCacheLocationPreConfigured())
{
Log.Info($"Deleting outputlocation: {Config.CacheLocation}");
try
{
Directory.Delete($"{Config.CacheLocation}", true);
}
catch (Exception ex)
{
Log.Exception($"{ex}");
}
}
Config.DisplayStatus();
Config.SaveConfigFile();
Log.Info($"{TotalFilesEnumerated} files enumerated.");
Log.Info($"{TotalFilesMatched} files matched.");
Log.Info($"{TotalFilesDownloaded} files downloaded.");
Log.Info($"{TotalFilesSkipped} files skipped.");
Log.Info($"{TotalFilesFormatted} files formatted.");
Log.Info($"{TotalErrors} errors.");
Log.Info($"{TotalRecords} records.");
Log.Info($"total execution time in minutes: { (DateTime.Now - StartTime).TotalMinutes.ToString("F2") }");
return(0);
}
catch (Exception ex)
{
Log.Exception($"{ex}");
return(1);
}
finally
{
Log.Close();
}
}
public int Collect(string[] args, ConfigurationOptions options)
{
try
{
if (!Config.PopulateConfig(args, options))
{
Config.SaveConfigFile();
return(1);
}
Log.Info($"version: {Version}");
_parallelConfig = new ParallelOptions {
MaxDegreeOfParallelism = Config.Threads
};
ServicePointManager.DefaultConnectionLimit = Config.Threads * MaxThreadMultiplier;
ServicePointManager.Expect100Continue = true;
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12;
ThreadPool.SetMinThreads(Config.Threads * MinThreadMultiplier, Config.Threads * MinThreadMultiplier);
ThreadPool.SetMaxThreads(Config.Threads * MaxThreadMultiplier, Config.Threads * MaxThreadMultiplier);
if (Config.NoProgressTimeoutMin > 0)
{
_noProgressTimer = new Timer(NoProgressCallback, null, 0, 60 * 1000);
}
if (!InitializeKusto() | !InitializeLogAnalytics())
{
return(1);
}
if (Config.SasEndpointInfo.IsPopulated())
{
DownloadAzureData();
}
else if (Config.IsCacheLocationPreConfigured())
{
UploadCacheData();
}
CustomTaskManager.WaitAll();
FinalizeKusto();
CustomTaskManager.Close();
if (Config.DeleteCache & Config.IsCacheLocationPreConfigured())
{
Log.Info($"Deleting outputlocation: {Config.CacheLocation}");
try
{
Directory.Delete($"{Config.CacheLocation}", true);
}
catch (Exception ex)
{
Log.Exception($"{ex}");
}
}
Config.DisplayStatus();
Config.SaveConfigFile();
TotalErrors += Log.LogErrors;
Log.Last($"{TotalFilesEnumerated} files enumerated.");
Log.Last($"{TotalFilesMatched} files matched.");
Log.Last($"{TotalFilesDownloaded} files downloaded.");
Log.Last($"{TotalFilesSkipped} files skipped.");
Log.Last($"{TotalFilesFormatted} files formatted.");
Log.Last($"{TotalErrors} errors.");
Log.Last($"{TotalRecords} records.");
if (TotalFilesEnumerated > 0)
{
if (Config.FileType != FileTypesEnum.table)
{
DateTime discoveredMinDateTime = new DateTime(DiscoveredMinDateTicks);
DateTime discoveredMaxDateTime = new DateTime(DiscoveredMaxDateTicks);
Log.Last($"discovered time range: {discoveredMinDateTime.ToString("o")} - {discoveredMaxDateTime.ToString("o")}", ConsoleColor.Green);
if (discoveredMinDateTime.Ticks > Config.EndTimeUtc.Ticks | discoveredMaxDateTime.Ticks < Config.StartTimeUtc.Ticks)
{
Log.Last($"error: configured time range not within discovered time range. configured time range: {Config.StartTimeUtc} - {Config.EndTimeUtc}", ConsoleColor.Red);
}
}
if (TotalFilesMatched + TotalRecords == 0 && (!string.IsNullOrEmpty(Config.UriFilter) | !string.IsNullOrEmpty(Config.ContainerFilter) | !string.IsNullOrEmpty(Config.NodeFilter)))
{
Log.Last("0 records found and filters are configured. verify filters and / or try time range are correct.", ConsoleColor.Yellow);
}
else if (TotalFilesMatched + TotalRecords == 0)
{
Log.Last("0 records found. verify time range is correct.", ConsoleColor.Yellow);
}
}
else
{
Log.Last("0 files enumerated.", ConsoleColor.Red);
}
Log.Last($"total execution time in minutes: { (DateTime.Now - StartTime).TotalMinutes.ToString("F2") }");
return(TotalErrors);
}
catch (Exception ex)
{
Log.Exception($"{ex}");
return(1);
}
finally
{
Log.Close();
}
}
