public static int Run(CompareOptions options) { var parameters = CreateParameters( (ConfigurationConstants.ExpectedUtterancesPathKey, options.ExpectedUtterancesPath), (ConfigurationConstants.ActualUtterancesPathKey, options.ActualUtterancesPath), (ConfigurationConstants.TestLabelKey, options.TestLabel)); var arguments = new List <string> { $"-p:{parameters}" }; if (options.OutputFolder != null) { arguments.Add($"--work={options.OutputFolder}"); } if (options.Metadata) { var expectedUtterances = Read <List <JsonLabeledUtterance> >(options.ExpectedUtterancesPath); var actualUtterances = Read <List <JsonLabeledUtterance> >(options.ActualUtterancesPath); var compareResults = TestCaseSource.GetNLUCompareResults(expectedUtterances, actualUtterances); var metadataPath = options.OutputFolder != null?Path.Combine(options.OutputFolder, TestMetadataFileName) : TestMetadataFileName; var statisticsPath = options.OutputFolder != null?Path.Combine(options.OutputFolder, TestStatisticsFileName) : TestStatisticsFileName; Write(metadataPath, compareResults.TestCases); File.WriteAllText(statisticsPath, JObject.FromObject(compareResults.Statistics).ToString()); compareResults.PrintResults(); } new AutoRun(typeof(ConfigurationConstants).Assembly).Execute(arguments.ToArray()); // We don't care if there are any failing NUnit tests return(0); }
private static int RunJson(CompareOptions options) { var expectedUtterances = Read <List <JsonLabeledUtterance> >(options.ExpectedUtterancesPath); var actualUtterances = Read <List <LabeledUtterance> >(options.ActualUtterancesPath); var testSettings = new TestSettings(options.TestSettingsPath, options.UnitTestMode); var compareResults = TestCaseSource.GetNLUCompareResults(expectedUtterances, actualUtterances, testSettings); var baseline = options.BaselinePath != null?Read <NLUStatistics>(options.BaselinePath) : null; compareResults.PrintResults(baseline); var metadataPath = options.OutputFolder != null?Path.Combine(options.OutputFolder, TestMetadataFileName) : TestMetadataFileName; var statisticsPath = options.OutputFolder != null?Path.Combine(options.OutputFolder, TestStatisticsFileName) : TestStatisticsFileName; Write(metadataPath, compareResults.TestCases); File.WriteAllText(statisticsPath, JObject.FromObject(compareResults.Statistics).ToString()); var failedThresholds = testSettings.Thresholds .Where(t => !compareResults.Statistics.CheckThreshold(baseline, t)) .ToList(); if (failedThresholds.Count > 0) { var failedThresholdsInfo = string.Join(", ", failedThresholds.Select(t => t.GetDescription())); Logger.LogWarning($"Performance threshold not met for {failedThresholdsInfo}."); } return(failedThresholds.Count); }