/// <summary> /// Gathers metrics for running model /// </summary> /// <param name="clientProcess">Process to run perf client with model.</param> public void GetMetricsRunningModel(Process clientProcess, MetricsCollector collector) { collector.ClearCachedMetrics(); while (IsRunning(clientProcess)) { collector.UpdateLatestMetrics(); Thread.Sleep(_UpdateFrequency); } }
/// <summary> /// Parses throughput from standard output /// </summary> /// <param name="clientProcess">Process to parse throughput from.</param> /// /// <param name="collector">MetricsCollector to update throughput for.</param> public void ParseThroughput(Process clientProcess, MetricsCollector collector) { while (!clientProcess.StandardOutput.EndOfStream) { var line = clientProcess.StandardOutput.ReadLine(); if (line.Contains("Throughput")) { var throughput = line.Split(":").Last().Trim(); collector.UpdateLatestMetricsThroughput(throughput); } Console.WriteLine(line); } }
/// <summary> /// Constructor for ModelAnalyzer /// Used for gathering and exporting metrics to screen /// </summary> /// <param name="config">Configuration object for Model Analyzer</param> /// <returns>Model Analyzer instance</returns> public ModelAnalyzer(ModelAnalyzerConfig config, MetricsCollector metricsCollectorServerOnly, MetricsCollector metricsCollectorModel) { _BatchSize = config.BatchSize; _ConcurrencyValues = config.ConcurrencyValues; _MetricsCollectorModel = metricsCollectorModel; _MetricsCollectorServerOnly = metricsCollectorServerOnly; _ModelFolder = config.ModelFolder; _ModelName = config.ModelName; _TritonVersion = config.TritonVersion; _MaxRetryTime = config.MaxRetryTime; _PerfClientBufferTime = config.PerfClientBufferTime; _UpdateFrequency = config.UpdateFrequency; }
/// <summary> /// Exports metrics from MetricsCollector to standard out and/or file /// </summary> /// <param name="options">Parsed options.</param> /// <param name="metricsCollectorServerOnly">Metrics collector for server only.</param> /// <param name="metricsCollectorModel">Metrics collector for model.</param> private static void ExportMetrics(Options options, MetricsCollector metricsCollectorServerOnly, MetricsCollector metricsCollectorModel) { //Write metrics to screen Console.WriteLine("\nServer Only:"); metricsCollectorServerOnly.ExportMetrics(); Console.WriteLine("\nModels:"); metricsCollectorModel.ExportMetrics(); //Write metrics to file if (options.ExportFlag) { metricsCollectorServerOnly.ExportMetrics(Path.Combine(options.ExportPath, options.FilenameServerOnly)); metricsCollectorModel.ExportMetrics(Path.Combine(options.ExportPath, options.FilenameModel)); } }
private static int Main(string[] args) { //Disable default help text. var parser = new CommandLine.Parser(with => with.HelpWriter = null); var program = new Program(); return(Parser.Default.ParseArguments <CLIOptions, K8sOptions>(args) .MapResult( (CLIOptions options) => { var analyzerConfig = program.ParseCLI(options); //Initialize metrics collectors var collectorConfig = new MetricsCollectorConfig() { RunLength = TimeSpan.FromSeconds(options.ServerMetricsDuration), UpdateFrequency = analyzerConfig.UpdateFrequency, }; var metricsCollectorServerOnly = new MetricsCollector(collectorConfig); var metricsCollectorModel = new MetricsCollector(collectorConfig); Console.CancelKeyPress += delegate { ExportMetrics(options, metricsCollectorServerOnly, metricsCollectorModel); }; try { foreach (var model in options.ModelNames) { try { while (ModelAnalyzer.IsServerRunning()) { Thread.Sleep(TimeSpan.FromSeconds(1)); } analyzerConfig.ModelName = model; var analyzer = new ModelAnalyzer(analyzerConfig, metricsCollectorServerOnly, metricsCollectorModel); analyzer.RunLocal(); } catch (ModelLoadException) { Console.WriteLine($"Failed to load {model} on inference server: skipping model"); } catch (Exception exception) { Console.WriteLine(exception.ToString()); } } } finally { ExportMetrics(options, metricsCollectorServerOnly, metricsCollectorModel); } return 0; }, (K8sOptions options) => { var analyzerConfig = program.ParseK8s(options); //Initialize metrics collectors var collectorConfig = new MetricsCollectorConfig() { RunLength = TimeSpan.FromSeconds(options.ServerMetricsDuration), UpdateFrequency = analyzerConfig.UpdateFrequency, }; var metricsCollectorServerOnly = new MetricsCollector(collectorConfig); var metricsCollectorModel = new MetricsCollector(collectorConfig); Console.CancelKeyPress += delegate { ExportMetrics(options, metricsCollectorServerOnly, metricsCollectorModel); }; try { foreach (var model in options.ModelNames) { try { analyzerConfig.ModelName = model; var analyzer = new ModelAnalyzer(analyzerConfig, metricsCollectorServerOnly, metricsCollectorModel); analyzer.RunK8s(); } catch (ModelLoadException) { Console.WriteLine($"Failed to load {model} on inference server: skipping model"); } catch (Exception exception) { Console.WriteLine(exception.ToString()); } } } finally { ExportMetrics(options, metricsCollectorServerOnly, metricsCollectorModel); } return 0; }, errs => { return -1; } )); }