public int Collect(List <string> uris = null)
{
try
{
if (!Initialize() || !InitializeKusto() || !InitializeLogAnalytics())
{
return(1);
}
if (Config.SasEndpointInfo.IsPopulated())
{
DownloadAzureData(uris);
}
else if (Config.IsCacheLocationPreConfigured())
{
UploadCacheData(uris);
}
CustomTaskManager.WaitAll();
FinalizeKusto();
if (Config.DeleteCache && Config.IsCacheLocationPreConfigured() && Directory.Exists(Config.CacheLocation))
{
Log.Info($"Deleting outputlocation: {Config.CacheLocation}");
try
{
Directory.Delete($"{Config.CacheLocation}", true);
}
catch (Exception ex)
{
Log.Exception($"{ex}");
}
}
Config.DisplayStatus();
Config.SaveConfigFile();
Instance.TotalErrors += Log.LogErrors;
LogSummary();
return(Instance.TotalErrors);
}
catch (Exception ex)
{
Log.Exception($"{ex}");
return(1);
}
finally
{
Log.Reset();
CustomTaskManager.Reset();
_noProgressTimer?.Dispose();
}
}
private static int ManageTasks(CustomTaskManager instance)
{
if (instance.RemoveWhenComplete && instance.AllTasks.Any(x => x.IsCompleted))
{
string taskUpdate = $"removing completed tasks from instance {instance.CallerName} " +
$"scheduled:{instance.AllTasks.Count()} " +
$"queued:{instance.QueuedTaskObjects.Count()}";
if (Config.LogDebug)
{
instance.AllTasks.ForEach(x => taskUpdate += $"\r\n ID:{x.Id.ToString().PadRight(6)} {x.Status.ToString().PadRight(15)} ({x.CreationOptions.ToString()})");
}
Log.Info(taskUpdate, ConsoleColor.White);
foreach (Task badTask in instance.AllTasks.FindAll(x => x.Status > TaskStatus.RanToCompletion))
{
Log.Error("task failure: ", badTask);
TotalErrors++;
}
instance.AllTasks.RemoveAll(x => x.IsCompleted);
}
while ((instance.CreationOptions == TaskCreationOptions.AttachedToParent | !instance.IsAboveQuota()) && instance.QueuedTaskObjects.Any())
{
TaskObject taskObject = instance.QueuedTaskObjects[0];
instance.ScheduleTask(taskObject);
instance.QueuedTaskObjects.Remove(taskObject);
Log.Highlight($"scheduled task {instance.GetHashCode()} " +
$"instance:{instance.CallerName} " +
$"total:{instance.AllTasks.Count()} " +
$"incomplete:{instance.AllTasks.Count(x => !x.IsCompleted)} " +
$"queued:{instance.QueuedTaskObjects.Count()} " +
$"GC:{GC.GetTotalMemory(false)} " +
$"total records:{TotalRecords} " +
$"rps:{TotalRecords / (DateTime.Now - StartTime).TotalSeconds}");
}
return(instance.AllTasks.Count(x => !x.IsCompleted));
}
public bool Initialize()
{
_noProgressTimer = new Timer(NoProgressCallback, null, 0, 60 * 1000);
Log.Open();
CustomTaskManager.Resume();
if (_initialized)
{
_taskManager?.Wait();
_taskManager = new CustomTaskManager();
Instance.Initialize();
}
else
{
if (!Config.PopulateConfig(_args))
{
Config.SaveConfigFile();
return(false);
}
_initialized = true;
Log.Info($"version: {Version}");
_parallelConfig = new ParallelOptions {
MaxDegreeOfParallelism = Config.Threads
};
ServicePointManager.DefaultConnectionLimit = Config.Threads * MaxThreadMultiplier;
ServicePointManager.Expect100Continue = true;
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12;
ThreadPool.SetMinThreads(Config.Threads * MinThreadMultiplier, Config.Threads * MinThreadMultiplier);
ThreadPool.SetMaxThreads(Config.Threads * MaxThreadMultiplier, Config.Threads * MaxThreadMultiplier);
}
return(true);
}
public int Execute(string[] args)
{
StartTime = DateTime.Now;
try
{
if (!Config.PopulateConfig(args))
{
Config.SaveConfigFile();
return(1);
}
Log.Info($"version: {Version}");
ParallelConfig = new ParallelOptions {
MaxDegreeOfParallelism = Config.Threads
};
ServicePointManager.DefaultConnectionLimit = Config.Threads * MaxThreadMultiplier;
ThreadPool.SetMinThreads(Config.Threads * MinThreadMultiplier, Config.Threads * MinThreadMultiplier);
ThreadPool.SetMaxThreads(Config.Threads * MaxThreadMultiplier, Config.Threads * MaxThreadMultiplier);
if (!InitializeKusto() | !InitializeLogAnalytics())
{
return(1);
}
if (Config.SasEndpointInfo.IsPopulated())
{
DownloadAzureData();
}
else if (Config.IsCacheLocationPreConfigured())
{
UploadCacheData();
}
CustomTaskManager.WaitAll();
FinalizeKusto();
CustomTaskManager.Close();
if (Config.DeleteCache & Config.IsCacheLocationPreConfigured())
{
Log.Info($"Deleting outputlocation: {Config.CacheLocation}");
try
{
Directory.Delete($"{Config.CacheLocation}", true);
}
catch (Exception ex)
{
Log.Exception($"{ex}");
}
}
Config.DisplayStatus();
Config.SaveConfigFile();
Log.Info($"{TotalFilesEnumerated} files enumerated.");
Log.Info($"{TotalFilesMatched} files matched.");
Log.Info($"{TotalFilesDownloaded} files downloaded.");
Log.Info($"{TotalFilesSkipped} files skipped.");
Log.Info($"{TotalFilesFormatted} files formatted.");
Log.Info($"{TotalErrors} errors.");
Log.Info($"{TotalRecords} records.");
Log.Info($"total execution time in minutes: { (DateTime.Now - StartTime).TotalMinutes.ToString("F2") }");
return(0);
}
catch (Exception ex)
{
Log.Exception($"{ex}");
return(1);
}
finally
{
Log.Close();
}
}
public int Collect(string[] args, ConfigurationOptions options)
{
try
{
if (!Config.PopulateConfig(args, options))
{
Config.SaveConfigFile();
return(1);
}
Log.Info($"version: {Version}");
_parallelConfig = new ParallelOptions {
MaxDegreeOfParallelism = Config.Threads
};
ServicePointManager.DefaultConnectionLimit = Config.Threads * MaxThreadMultiplier;
ServicePointManager.Expect100Continue = true;
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12;
ThreadPool.SetMinThreads(Config.Threads * MinThreadMultiplier, Config.Threads * MinThreadMultiplier);
ThreadPool.SetMaxThreads(Config.Threads * MaxThreadMultiplier, Config.Threads * MaxThreadMultiplier);
if (Config.NoProgressTimeoutMin > 0)
{
_noProgressTimer = new Timer(NoProgressCallback, null, 0, 60 * 1000);
}
if (!InitializeKusto() | !InitializeLogAnalytics())
{
return(1);
}
if (Config.SasEndpointInfo.IsPopulated())
{
DownloadAzureData();
}
else if (Config.IsCacheLocationPreConfigured())
{
UploadCacheData();
}
CustomTaskManager.WaitAll();
FinalizeKusto();
CustomTaskManager.Close();
if (Config.DeleteCache & Config.IsCacheLocationPreConfigured())
{
Log.Info($"Deleting outputlocation: {Config.CacheLocation}");
try
{
Directory.Delete($"{Config.CacheLocation}", true);
}
catch (Exception ex)
{
Log.Exception($"{ex}");
}
}
Config.DisplayStatus();
Config.SaveConfigFile();
TotalErrors += Log.LogErrors;
Log.Last($"{TotalFilesEnumerated} files enumerated.");
Log.Last($"{TotalFilesMatched} files matched.");
Log.Last($"{TotalFilesDownloaded} files downloaded.");
Log.Last($"{TotalFilesSkipped} files skipped.");
Log.Last($"{TotalFilesFormatted} files formatted.");
Log.Last($"{TotalErrors} errors.");
Log.Last($"{TotalRecords} records.");
if (TotalFilesEnumerated > 0)
{
if (Config.FileType != FileTypesEnum.table)
{
DateTime discoveredMinDateTime = new DateTime(DiscoveredMinDateTicks);
DateTime discoveredMaxDateTime = new DateTime(DiscoveredMaxDateTicks);
Log.Last($"discovered time range: {discoveredMinDateTime.ToString("o")} - {discoveredMaxDateTime.ToString("o")}", ConsoleColor.Green);
if (discoveredMinDateTime.Ticks > Config.EndTimeUtc.Ticks | discoveredMaxDateTime.Ticks < Config.StartTimeUtc.Ticks)
{
Log.Last($"error: configured time range not within discovered time range. configured time range: {Config.StartTimeUtc} - {Config.EndTimeUtc}", ConsoleColor.Red);
}
}
if (TotalFilesMatched + TotalRecords == 0 && (!string.IsNullOrEmpty(Config.UriFilter) | !string.IsNullOrEmpty(Config.ContainerFilter) | !string.IsNullOrEmpty(Config.NodeFilter)))
{
Log.Last("0 records found and filters are configured. verify filters and / or try time range are correct.", ConsoleColor.Yellow);
}
else if (TotalFilesMatched + TotalRecords == 0)
{
Log.Last("0 records found. verify time range is correct.", ConsoleColor.Yellow);
}
}
else
{
Log.Last("0 files enumerated.", ConsoleColor.Red);
}
Log.Last($"total execution time in minutes: { (DateTime.Now - StartTime).TotalMinutes.ToString("F2") }");
return(TotalErrors);
}
catch (Exception ex)
{
Log.Exception($"{ex}");
return(1);
}
finally
{
Log.Close();
}
}