/// <summary>
/// Gathers metrics for running model
/// </summary>
/// <param name="clientProcess">Process to run perf client with model.</param>
public void GetMetricsRunningModel(Process clientProcess, MetricsCollector collector)
{
collector.ClearCachedMetrics();
while (IsRunning(clientProcess))
{
collector.UpdateLatestMetrics();
Thread.Sleep(_UpdateFrequency);
}
}
/// <summary>
/// Parses throughput from standard output
/// </summary>
/// <param name="clientProcess">Process to parse throughput from.</param>
/// /// <param name="collector">MetricsCollector to update throughput for.</param>
public void ParseThroughput(Process clientProcess, MetricsCollector collector)
{
while (!clientProcess.StandardOutput.EndOfStream)
{
var line = clientProcess.StandardOutput.ReadLine();
if (line.Contains("Throughput"))
{
var throughput = line.Split(":").Last().Trim();
collector.UpdateLatestMetricsThroughput(throughput);
}
Console.WriteLine(line);
}
}
/// <summary>
/// Constructor for ModelAnalyzer
/// Used for gathering and exporting metrics to screen
/// </summary>
/// <param name="config">Configuration object for Model Analyzer</param>
/// <returns>Model Analyzer instance</returns>
public ModelAnalyzer(ModelAnalyzerConfig config, MetricsCollector metricsCollectorServerOnly, MetricsCollector metricsCollectorModel)
{
_BatchSize = config.BatchSize;
_ConcurrencyValues = config.ConcurrencyValues;
_MetricsCollectorModel = metricsCollectorModel;
_MetricsCollectorServerOnly = metricsCollectorServerOnly;
_ModelFolder = config.ModelFolder;
_ModelName = config.ModelName;
_TritonVersion = config.TritonVersion;
_MaxRetryTime = config.MaxRetryTime;
_PerfClientBufferTime = config.PerfClientBufferTime;
_UpdateFrequency = config.UpdateFrequency;
}
/// <summary>
/// Exports metrics from MetricsCollector to standard out and/or file
/// </summary>
/// <param name="options">Parsed options.</param>
/// <param name="metricsCollectorServerOnly">Metrics collector for server only.</param>
/// <param name="metricsCollectorModel">Metrics collector for model.</param>
private static void ExportMetrics(Options options, MetricsCollector metricsCollectorServerOnly, MetricsCollector metricsCollectorModel)
{
//Write metrics to screen
Console.WriteLine("\nServer Only:");
metricsCollectorServerOnly.ExportMetrics();
Console.WriteLine("\nModels:");
metricsCollectorModel.ExportMetrics();
//Write metrics to file
if (options.ExportFlag)
{
metricsCollectorServerOnly.ExportMetrics(Path.Combine(options.ExportPath, options.FilenameServerOnly));
metricsCollectorModel.ExportMetrics(Path.Combine(options.ExportPath, options.FilenameModel));
}
}
private static int Main(string[] args)
{
//Disable default help text.
var parser = new CommandLine.Parser(with => with.HelpWriter = null);
var program = new Program();
return(Parser.Default.ParseArguments <CLIOptions, K8sOptions>(args)
.MapResult(
(CLIOptions options) =>
{
var analyzerConfig = program.ParseCLI(options);
//Initialize metrics collectors
var collectorConfig = new MetricsCollectorConfig()
{
RunLength = TimeSpan.FromSeconds(options.ServerMetricsDuration),
UpdateFrequency = analyzerConfig.UpdateFrequency,
};
var metricsCollectorServerOnly = new MetricsCollector(collectorConfig);
var metricsCollectorModel = new MetricsCollector(collectorConfig);
Console.CancelKeyPress += delegate
{
ExportMetrics(options, metricsCollectorServerOnly, metricsCollectorModel);
};
try
{
foreach (var model in options.ModelNames)
{
try
{
while (ModelAnalyzer.IsServerRunning())
{
Thread.Sleep(TimeSpan.FromSeconds(1));
}
analyzerConfig.ModelName = model;
var analyzer = new ModelAnalyzer(analyzerConfig, metricsCollectorServerOnly, metricsCollectorModel);
analyzer.RunLocal();
}
catch (ModelLoadException)
{
Console.WriteLine($"Failed to load {model} on inference server: skipping model");
}
catch (Exception exception)
{
Console.WriteLine(exception.ToString());
}
}
}
finally
{
ExportMetrics(options, metricsCollectorServerOnly, metricsCollectorModel);
}
return 0;
},
(K8sOptions options) =>
{
var analyzerConfig = program.ParseK8s(options);
//Initialize metrics collectors
var collectorConfig = new MetricsCollectorConfig()
{
RunLength = TimeSpan.FromSeconds(options.ServerMetricsDuration),
UpdateFrequency = analyzerConfig.UpdateFrequency,
};
var metricsCollectorServerOnly = new MetricsCollector(collectorConfig);
var metricsCollectorModel = new MetricsCollector(collectorConfig);
Console.CancelKeyPress += delegate
{
ExportMetrics(options, metricsCollectorServerOnly, metricsCollectorModel);
};
try
{
foreach (var model in options.ModelNames)
{
try
{
analyzerConfig.ModelName = model;
var analyzer = new ModelAnalyzer(analyzerConfig, metricsCollectorServerOnly, metricsCollectorModel);
analyzer.RunK8s();
}
catch (ModelLoadException)
{
Console.WriteLine($"Failed to load {model} on inference server: skipping model");
}
catch (Exception exception)
{
Console.WriteLine(exception.ToString());
}
}
}
finally
{
ExportMetrics(options, metricsCollectorServerOnly, metricsCollectorModel);
}
return 0;
},
errs =>
{
return -1;
}
));
}
