/// <summary>
/// Parses throughput from standard output
/// </summary>
/// <param name="clientProcess">Process to parse throughput from.</param>
/// /// <param name="collector">MetricsCollector to update throughput for.</param>
public void ParseThroughput(Process clientProcess, MetricsCollector collector)
{
while (!clientProcess.StandardOutput.EndOfStream)
{
var line = clientProcess.StandardOutput.ReadLine();
if (line.Contains("Throughput"))
{
var throughput = line.Split(":").Last().Trim();
collector.UpdateLatestMetricsThroughput(throughput);
}
Console.WriteLine(line);
}
}
/// <summary>
/// High-level function running Model Analyzer
/// </summary>
/// <param name="clientProcess">Client process to run, excluding batch and concurrency values.</param>
public void Run(Process clientProcess)
{
HideProcessInput(clientProcess);
// Steps:
// 1. Verify that Triton is running.
// 2. Gather server-only metrics, if they have not yet been gathered.
// 3. Use Triton API to load model into server.
// 4. Use Perf Client to load model into memory.
// 5. For each concurrency and batch size configuration, run the model in Perf Client.
// 6. As each configuration is running, gather metrics..
// 7. Triton server and client's associated processes and containers are cleaned up.
WaitUntilPortReady();
if (_MetricsCollectorServerOnly._MetricsSamples.Count == 0)
{
_MetricsCollectorServerOnly.AddNewSample(ServerName, 0, 0);
_MetricsCollectorServerOnly.UpdateLatestMetricsThroughput("0 infer/sec");
_MetricsCollectorServerOnly.GenerateMetrics();
}
LoadModel();
var baseArgs = clientProcess.StartInfo.Arguments;
//Use Perf Client to load model into memory.
HideProcessOutput(clientProcess);
RunModel(clientProcess);
clientProcess.WaitForExit();
foreach (var batch in _BatchSize)
{
if (batch < 0)
{
Console.WriteLine($"Invalid batch size: {batch}: skipping");
continue;
}
foreach (var concurrency in _ConcurrencyValues)
{
if (concurrency < 0)
{
Console.WriteLine($"Invalid concurrency value: {concurrency}: skipping");
continue;
}
_MetricsCollectorModel.AddNewSample(_ModelName, batch, concurrency);
try
{
clientProcess.StartInfo.Arguments = string.Concat(baseArgs,
$"--percentile=96 -b {batch} --concurrency-range {concurrency}:{concurrency}");
Console.WriteLine($"\n{_ModelName}");
RunModel(clientProcess);
Task.Run(
() => ParseThroughput(clientProcess, _MetricsCollectorModel));
var parseError = new Task(
() => ParseError(clientProcess));
parseError.Start();
GetMetricsRunningModel(clientProcess, _MetricsCollectorModel);
parseError.Wait();
KillProcess(clientProcess);
}
catch (Exception exception)
{
_MetricsCollectorModel.DeletesLatestSample();
Console.WriteLine(exception.Message);
}
}
}
}