/// <summary>
/// Constructor for MetricsCollector
/// </summary>
public MetricsCollector(MetricsCollectorConfig config)
{
_RunLength = config.RunLength;
_UpdateFrequency = config.UpdateFrequency;
_Watcher = _GpuMetrics.GetWatcher("metrics", _UpdateFrequency);
}
private static int Main(string[] args)
{
//Disable default help text.
var parser = new CommandLine.Parser(with => with.HelpWriter = null);
var program = new Program();
return(Parser.Default.ParseArguments <CLIOptions, K8sOptions>(args)
.MapResult(
(CLIOptions options) =>
{
var analyzerConfig = program.ParseCLI(options);
//Initialize metrics collectors
var collectorConfig = new MetricsCollectorConfig()
{
RunLength = TimeSpan.FromSeconds(options.ServerMetricsDuration),
UpdateFrequency = analyzerConfig.UpdateFrequency,
};
var metricsCollectorServerOnly = new MetricsCollector(collectorConfig);
var metricsCollectorModel = new MetricsCollector(collectorConfig);
Console.CancelKeyPress += delegate
{
ExportMetrics(options, metricsCollectorServerOnly, metricsCollectorModel);
};
try
{
foreach (var model in options.ModelNames)
{
try
{
while (ModelAnalyzer.IsServerRunning())
{
Thread.Sleep(TimeSpan.FromSeconds(1));
}
analyzerConfig.ModelName = model;
var analyzer = new ModelAnalyzer(analyzerConfig, metricsCollectorServerOnly, metricsCollectorModel);
analyzer.RunLocal();
}
catch (ModelLoadException)
{
Console.WriteLine($"Failed to load {model} on inference server: skipping model");
}
catch (Exception exception)
{
Console.WriteLine(exception.ToString());
}
}
}
finally
{
ExportMetrics(options, metricsCollectorServerOnly, metricsCollectorModel);
}
return 0;
},
(K8sOptions options) =>
{
var analyzerConfig = program.ParseK8s(options);
//Initialize metrics collectors
var collectorConfig = new MetricsCollectorConfig()
{
RunLength = TimeSpan.FromSeconds(options.ServerMetricsDuration),
UpdateFrequency = analyzerConfig.UpdateFrequency,
};
var metricsCollectorServerOnly = new MetricsCollector(collectorConfig);
var metricsCollectorModel = new MetricsCollector(collectorConfig);
Console.CancelKeyPress += delegate
{
ExportMetrics(options, metricsCollectorServerOnly, metricsCollectorModel);
};
try
{
foreach (var model in options.ModelNames)
{
try
{
analyzerConfig.ModelName = model;
var analyzer = new ModelAnalyzer(analyzerConfig, metricsCollectorServerOnly, metricsCollectorModel);
analyzer.RunK8s();
}
catch (ModelLoadException)
{
Console.WriteLine($"Failed to load {model} on inference server: skipping model");
}
catch (Exception exception)
{
Console.WriteLine(exception.ToString());
}
}
}
finally
{
ExportMetrics(options, metricsCollectorServerOnly, metricsCollectorModel);
}
return 0;
},
errs =>
{
return -1;
}
));
}