/// <summary>
 /// Constructor for MetricsCollector
 /// </summary>
 public MetricsCollector(MetricsCollectorConfig config)
 {
     _RunLength       = config.RunLength;
     _UpdateFrequency = config.UpdateFrequency;
     _Watcher         = _GpuMetrics.GetWatcher("metrics", _UpdateFrequency);
 }
Example #2
0
        private static int Main(string[] args)
        {
            //Disable default help text.
            var parser = new CommandLine.Parser(with => with.HelpWriter = null);

            var program = new Program();

            return(Parser.Default.ParseArguments <CLIOptions, K8sOptions>(args)
                   .MapResult(
                       (CLIOptions options) =>
            {
                var analyzerConfig = program.ParseCLI(options);

                //Initialize metrics collectors
                var collectorConfig = new MetricsCollectorConfig()
                {
                    RunLength = TimeSpan.FromSeconds(options.ServerMetricsDuration),
                    UpdateFrequency = analyzerConfig.UpdateFrequency,
                };

                var metricsCollectorServerOnly = new MetricsCollector(collectorConfig);
                var metricsCollectorModel = new MetricsCollector(collectorConfig);

                Console.CancelKeyPress += delegate
                {
                    ExportMetrics(options, metricsCollectorServerOnly, metricsCollectorModel);
                };

                try
                {
                    foreach (var model in options.ModelNames)
                    {
                        try
                        {
                            while (ModelAnalyzer.IsServerRunning())
                            {
                                Thread.Sleep(TimeSpan.FromSeconds(1));
                            }

                            analyzerConfig.ModelName = model;
                            var analyzer = new ModelAnalyzer(analyzerConfig, metricsCollectorServerOnly, metricsCollectorModel);
                            analyzer.RunLocal();
                        }
                        catch (ModelLoadException)
                        {
                            Console.WriteLine($"Failed to load {model} on inference server: skipping model");
                        }
                        catch (Exception exception)
                        {
                            Console.WriteLine(exception.ToString());
                        }
                    }
                }
                finally
                {
                    ExportMetrics(options, metricsCollectorServerOnly, metricsCollectorModel);
                }

                return 0;
            },
                       (K8sOptions options) =>
            {
                var analyzerConfig = program.ParseK8s(options);

                //Initialize metrics collectors
                var collectorConfig = new MetricsCollectorConfig()
                {
                    RunLength = TimeSpan.FromSeconds(options.ServerMetricsDuration),
                    UpdateFrequency = analyzerConfig.UpdateFrequency,
                };

                var metricsCollectorServerOnly = new MetricsCollector(collectorConfig);
                var metricsCollectorModel = new MetricsCollector(collectorConfig);

                Console.CancelKeyPress += delegate
                {
                    ExportMetrics(options, metricsCollectorServerOnly, metricsCollectorModel);
                };

                try
                {
                    foreach (var model in options.ModelNames)
                    {
                        try
                        {
                            analyzerConfig.ModelName = model;
                            var analyzer = new ModelAnalyzer(analyzerConfig, metricsCollectorServerOnly, metricsCollectorModel);
                            analyzer.RunK8s();
                        }
                        catch (ModelLoadException)
                        {
                            Console.WriteLine($"Failed to load {model} on inference server: skipping model");
                        }
                        catch (Exception exception)
                        {
                            Console.WriteLine(exception.ToString());
                        }
                    }
                }
                finally
                {
                    ExportMetrics(options, metricsCollectorServerOnly, metricsCollectorModel);
                }

                return 0;
            },
                       errs =>
            {
                return -1;
            }
                       ));
        }