/// <summary> /// Logs the test pass/fail message. /// </summary> /// <param name="turnResult">Resulting dialog turn.</param> private void DisplayTestResultMessage(TurnResult turnResult) { if (turnResult.Pass) { Trace.TraceInformation($"Turn passed (DialogId {this.DialogID}, TurnID {turnResult.TurnID})"); } else { string failMessage = $"Turn failed (DialogId {this.DialogID}, TurnID {turnResult.TurnID}) due to: "; bool commaNeeded = false; if (!turnResult.ResponseLatencyMatch) { failMessage += "latency mismatch"; commaNeeded = true; } if (!turnResult.ResponseMatch) { if (commaNeeded) { failMessage += ", "; } failMessage += "bot response mismatch"; commaNeeded = true; } if (!turnResult.UtteranceMatch) { if (commaNeeded) { failMessage += ", "; } failMessage += "utterance mismatch"; commaNeeded = true; } if (!turnResult.TTSAudioResponseDurationMatch) { if (commaNeeded) { failMessage += ", "; } failMessage += "TTS audio duration mismatch"; } Trace.TraceInformation(failMessage); } }
private static async Task <bool> ProcessTestFiles(AppSettings appSettings) { List <TestReport> allInputFilesTestReport = new List <TestReport>(); bool testPass = true; if (!string.IsNullOrEmpty(appSettings.AriaProjectKey)) { #if USE_ARIA_LOGGING AriaLogger.Start(appSettings.AriaProjectKey); #endif } foreach (TestSettings tests in appSettings.Tests) { bool isFirstDialog = true; bool connectionEstablished = false; BotConnector botConnector = null; Trace.IndentLevel = 0; if (tests.Skip) { Trace.TraceInformation($"Skipping file {tests.FileName}"); continue; } else { Trace.TraceInformation($"Processing file {tests.FileName}"); } string inputFileName = Path.Combine(appSettings.InputFolder, tests.FileName); string testName = Path.GetFileNameWithoutExtension(inputFileName); if (string.IsNullOrEmpty(appSettings.OutputFolder)) { appSettings.OutputFolder = Directory.GetCurrentDirectory(); } string outputPath = Path.Combine(appSettings.OutputFolder, testName + "Output"); DirectoryInfo outputDirectory = Directory.CreateDirectory(outputPath); string outputFileName = Path.Combine(outputDirectory.FullName, testName + "Output.json"); StreamReader file = new StreamReader(inputFileName, Encoding.UTF8); string txt = file.ReadToEnd(); file.Close(); var fileContents = JsonConvert.DeserializeObject <List <Dialog> >(txt); // Keep track of the detailed dialog results for all dialogs in a single input test file. This list will be serialized to JSON as the output for this test file. List <DialogResult> dialogResults = new List <DialogResult>(); // Keep track of high-level (summary) results for all dialogs in a single input test file. This list will be serialized to JSON as part of the overall single test report. List <DialogReport> dialogReports = new List <DialogReport>(); foreach (Dialog dialog in fileContents) { Trace.IndentLevel = 1; if (dialog.Skip) { Trace.TraceInformation($"Skipping DialogID {dialog.DialogID}"); continue; } else { Trace.TraceInformation($"[{DateTime.Now.ToString("h:mm:ss tt", CultureInfo.CurrentCulture)}] Running DialogId {dialog.DialogID}, description \"{dialog.Description}\""); } // Capture and compute the output for this dialog in this variable. DialogResult dialogResult = new DialogResult(appSettings, dialog.DialogID, dialog.Description); // Capture outputs of all turns in this dialog in this list. List <TurnResult> turnResults = new List <TurnResult>(); // Keep track of turn pass/fail : per turn. List <bool> turnPassResults = new List <bool>(); if (isFirstDialog || tests.SingleConnection == false) { // Always establish a connection with the bot for the first dialog in the test file. // If SingleConnection is false, it also means we need to re-establish the connection before each of the dialogs in the test file. if (botConnector != null) { await botConnector.Disconnect().ConfigureAwait(false); botConnector.Dispose(); } connectionEstablished = true; botConnector = new BotConnector(); botConnector.InitConnector(appSettings); await botConnector.Connect().ConfigureAwait(false); } isFirstDialog = false; foreach (Turn turn in dialog.Turns) { // Application crashes in a multi-turn dialog with Keyword in each Turn // Crash occurs when calling StartKeywordRecognitionAsync after calling StopKeywordRecognitionAsync in the previous Turn. // In order to avoid this crash, only have Keyword in Turn 0 of a multi-turn Keyword containing Dialog. // This is being investigated. // MS-Internal bug number: 2300634. // https://msasg.visualstudio.com/Skyman/_workitems/edit/2300634/ if (turn.Keyword) { await botConnector.StartKeywordRecognitionAsync().ConfigureAwait(false); } Trace.IndentLevel = 2; Trace.TraceInformation($"[{DateTime.Now.ToString("h:mm:ss tt", CultureInfo.CurrentCulture)}] Running Turn {turn.TurnID}"); Trace.IndentLevel = 3; if (turn.Sleep > 0) { Trace.TraceInformation($"Sleeping for {turn.Sleep} msec"); System.Threading.Thread.Sleep(turn.Sleep); } int responseCount = 0; bool bootstrapMode = true; if (turn.ExpectedResponses != null && turn.ExpectedResponses.Count != 0) { responseCount = turn.ExpectedResponses.Count; bootstrapMode = false; } botConnector.SetInputValues(testName, dialog.DialogID, turn.TurnID, responseCount, tests.IgnoreActivities, turn.Keyword); // Send up WAV File if present if (!string.IsNullOrEmpty(turn.WAVFile)) { botConnector.SendAudio(turn.WAVFile); } // Send up Utterance if present else if (!string.IsNullOrEmpty(turn.Utterance)) { botConnector = await botConnector.Send(turn.Utterance).ConfigureAwait(false); } // Send up activity if configured else if (!string.IsNullOrEmpty(turn.Activity)) { botConnector = await botConnector.SendActivity(turn.Activity).ConfigureAwait(false); } // All bot reply activities are captured in this variable. dialogResult.BotResponses = botConnector.WaitAndProcessBotReplies(bootstrapMode); // Capture the result of this turn in this variable and validate the turn. TurnResult turnResult = dialogResult.BuildOutput(turn, bootstrapMode, botConnector.RecognizedText, botConnector.RecognizedKeyword); if (!dialogResult.ValidateTurn(turnResult, bootstrapMode)) { testPass = false; } // Add the turn result to the list of turn results. turnResults.Add(turnResult); // Add the turn completion status to the list of turn completions. turnPassResults.Add(turnResult.Pass); if (turn.Keyword) { await botConnector.StopKeywordRecognitionAsync().ConfigureAwait(false); } } // End of turns loop dialogResult.Turns = turnResults; dialogResults.Add(dialogResult); DialogReport dialogReport = new DialogReport(dialogResult.DialogID, dialog.Description, turnPassResults); dialogReports.Add(dialogReport); turnPassResults = new List <bool>(); Trace.IndentLevel = 1; if (dialogReport.DialogPass) { Trace.TraceInformation($"DialogId {dialog.DialogID} passed"); #if USE_ARIA_LOGGING AriaLogger.Log(AriaLogger.EventNameDialogSucceeded, dialog.DialogID, dialog.Description); #endif } else { Trace.TraceInformation($"DialogId {dialog.DialogID} failed"); #if USE_ARIA_LOGGING AriaLogger.Log(AriaLogger.EventNameDialogFailed, dialog.DialogID, dialog.Description); #endif } } // End of dialog loop TestReport fileTestReport = new TestReport { FileName = inputFileName, DialogReports = dialogReports, DialogCount = dialogReports.Count, }; fileTestReport.ComputeDialogPassRate(); allInputFilesTestReport.Add(fileTestReport); File.WriteAllText(outputFileName, JsonConvert.SerializeObject(dialogResults, Formatting.Indented, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore })); if (connectionEstablished) { await botConnector.Disconnect().ConfigureAwait(false); botConnector.Dispose(); } } // End of inputFiles loop File.WriteAllText(Path.Combine(appSettings.OutputFolder, ProgramConstants.TestReportFileName), JsonConvert.SerializeObject(allInputFilesTestReport, Formatting.Indented)); Trace.IndentLevel = 0; if (testPass) { Trace.TraceInformation("********** TEST PASS **********"); } else { Trace.TraceInformation("********** TEST FAILED **********"); } #if USE_ARIA_LOGGING AriaLogger.Stop(); #endif return(testPass); }
private static async Task <bool> ProcessDialog( List <Dialog> fileContents, BotConnector botConnector, AppSettings appSettings, bool isFirstDialog, TestSettings tests, bool connectionEstablished, string testName, List <DialogReport> dialogReports, bool testPass, List <DialogResult> dialogResults, bool sendFirst) { foreach (Dialog dialog in fileContents) { Trace.IndentLevel = 1; if (dialog.Skip) { Trace.TraceInformation($"Skipping DialogID {dialog.DialogID}"); continue; } else { Trace.TraceInformation($"[{DateTime.Now.ToString("hh:mm:ss.fff", CultureInfo.CurrentCulture)}] Running DialogId {dialog.DialogID}, description \"{dialog.Description}\""); } // Capture and compute the output for this dialog in this variable. DialogResult dialogResult = new DialogResult(appSettings, dialog.DialogID, dialog.Description); // Capture outputs of all turns in this dialog in this list. List <TurnResult> turnResults = new List <TurnResult>(); // Keep track of turn pass/fail : per turn. List <bool> turnPassResults = new List <bool>(); if (isFirstDialog || tests.SingleConnection == false) { // Always establish a connection with the bot for the first dialog in the test file. // If SingleConnection is false, it also means we need to re-establish the connection before each of the dialogs in the test file. if (botConnector != null) { await botConnector.Disconnect().ConfigureAwait(false); botConnector.Dispose(); } connectionEstablished = true; botConnector = new BotConnector(); botConnector.InitConnector(appSettings); await botConnector.Connect().ConfigureAwait(false); } isFirstDialog = false; foreach (Turn turn in dialog.Turns) { // Application crashes in a multi-turn dialog with Keyword in each Turn // Crash occurs when calling StartKeywordRecognitionAsync after calling StopKeywordRecognitionAsync in the previous Turn. // In order to avoid this crash, only have Keyword in Turn 0 of a multi-turn Keyword containing Dialog. // This is being investigated. // MS-Internal bug number: 2300634. // https://msasg.visualstudio.com/Skyman/_workitems/edit/2300634/ if (turn.Keyword) { await botConnector.StartKeywordRecognitionAsync().ConfigureAwait(false); } Trace.IndentLevel = 2; Trace.TraceInformation($"[{DateTime.Now.ToString("hh:mm:ss.fff", CultureInfo.CurrentCulture)}] Running Turn {turn.TurnID}"); Trace.IndentLevel = 3; if (turn.Sleep > 0) { Trace.TraceInformation($"Sleeping for {turn.Sleep} msec"); System.Threading.Thread.Sleep(turn.Sleep); } int responseCount = 0; bool bootstrapMode = true; if (turn.ExpectedResponses != null && turn.ExpectedResponses.Count != 0) { responseCount = turn.ExpectedResponses.Count; bootstrapMode = false; } botConnector.SetInputValues(testName, dialog.DialogID, turn.TurnID, responseCount, tests.IgnoreActivities, turn.Keyword); if (tests.WavAndUtterancePairs && !string.IsNullOrWhiteSpace(turn.WAVFile) && !string.IsNullOrWhiteSpace(turn.Utterance)) { // Send up WAV File if present if (!string.IsNullOrEmpty(turn.WAVFile) && sendFirst) { botConnector.SendAudio(turn.WAVFile); } // Send up Utterance if present else if (!string.IsNullOrEmpty(turn.Utterance) && !sendFirst) { botConnector = await botConnector.Send(turn.Utterance).ConfigureAwait(false); } } // WavAndUtterancePair is false send either wavfile or utterance if present. else if (!tests.WavAndUtterancePairs) { // Send up WAV File if present if (!string.IsNullOrEmpty(turn.WAVFile)) { botConnector.SendAudio(turn.WAVFile); } // Send up Utterance if present else if (!string.IsNullOrEmpty(turn.Utterance)) { botConnector = await botConnector.Send(turn.Utterance).ConfigureAwait(false); } } // Send up activity if configured else if (!string.IsNullOrEmpty(turn.Activity)) { botConnector = await botConnector.SendActivity(turn.Activity).ConfigureAwait(false); } // All bot reply activities are captured in this variable. dialogResult.BotResponses = botConnector.WaitAndProcessBotReplies(bootstrapMode); // Capture the result of this turn in this variable and validate the turn. TurnResult turnResult = dialogResult.BuildOutput(turn, bootstrapMode, botConnector.RecognizedText, botConnector.RecognizedKeyword); if (!dialogResult.ValidateTurn(turnResult, bootstrapMode)) { testPass = false; } // Add the turn result to the list of turn results. turnResults.Add(turnResult); // Add the turn completion status to the list of turn completions. turnPassResults.Add(turnResult.Pass); if (turn.Keyword) { await botConnector.StopKeywordRecognitionAsync().ConfigureAwait(false); } } // End of turns loop dialogResult.Turns = turnResults; dialogResults.Add(dialogResult); DialogReport dialogReport = new DialogReport(dialogResult.DialogID, dialog.Description, turnPassResults); dialogReports.Add(dialogReport); turnPassResults = new List <bool>(); Trace.IndentLevel = 1; if (dialogReport.DialogPass) { Trace.TraceInformation($"DialogId {dialog.DialogID} passed"); #if USE_ARIA_LOGGING AriaLogger.Log(AriaLogger.EventNameDialogSucceeded, dialog.DialogID, dialog.Description); #endif } else { Trace.TraceInformation($"DialogId {dialog.DialogID} failed"); #if USE_ARIA_LOGGING AriaLogger.Log(AriaLogger.EventNameDialogFailed, dialog.DialogID, dialog.Description); #endif } } // End of dialog loop return(testPass); }
/// <summary> /// Returns bool values indicating if Actual parameters match expected parameters. /// Parameters include Intents, slots, responses, TTS duration, and latency. /// </summary> /// <param name="turnResult">TurnResult object capturing the responses from the bot for this turn.</param> /// <param name="bootstrapMode"> Boolean which defines if turn is in bootstrapping mode or not.</param> /// <returns>true if the all turn tests pass, false if any of the test failed.</returns> public bool ValidateTurn(TurnResult turnResult, bool bootstrapMode) { turnResult.ResponseMatch = true; turnResult.UtteranceMatch = true; turnResult.TTSAudioResponseDurationMatch = true; turnResult.ResponseLatencyMatch = true; turnResult.Pass = true; int margin = this.appSettings.TTSAudioDurationMargin; if (!bootstrapMode) { turnResult.ResponseMatch = DialogResult.ActivityListsMatch(turnResult.ExpectedResponses, turnResult.ActualResponses); if (!string.IsNullOrWhiteSpace(turnResult.WAVFile) && !string.IsNullOrWhiteSpace(turnResult.Utterance)) { string normalizedActualRecognizedText = new string(turnResult.ActualRecognizedText.Where(c => !char.IsPunctuation(c) && !char.IsWhiteSpace(c)).ToArray()).ToUpperInvariant(); string normalizedExpectedRecognizedText = new string(turnResult.Utterance.Where(c => !char.IsPunctuation(c) && !char.IsWhiteSpace(c)).ToArray()).ToUpperInvariant(); if (!normalizedExpectedRecognizedText.Equals(normalizedActualRecognizedText, StringComparison.OrdinalIgnoreCase)) { Trace.TraceInformation($"Recognized text \"{turnResult.ActualRecognizedText}\" does not match \"{turnResult.Utterance}\""); turnResult.UtteranceMatch = false; } } bool durationMatch = true; if (turnResult.ExpectedTTSAudioResponseDuration != null && turnResult.ExpectedTTSAudioResponseDuration.Count != 0 && turnResult.ActualTTSAudioResponseDuration != null && turnResult.ActualTTSAudioResponseDuration.Count != 0) { if (turnResult.ExpectedTTSAudioResponseDuration.Count > turnResult.ActualTTSAudioResponseDuration.Count) { turnResult.TTSAudioResponseDurationMatch = false; } else { for (int i = 0; i < turnResult.ExpectedTTSAudioResponseDuration.Count; i++) { if (turnResult.ExpectedTTSAudioResponseDuration[i] > 0) { if (turnResult.ActualTTSAudioResponseDuration[i] >= (turnResult.ExpectedTTSAudioResponseDuration[i] - margin) && turnResult.ActualTTSAudioResponseDuration[i] <= (turnResult.ExpectedTTSAudioResponseDuration[i] + margin)) { if (durationMatch) { turnResult.TTSAudioResponseDurationMatch = true; } } else { Trace.TraceInformation($"Actual TTS audio duration {turnResult.ActualTTSAudioResponseDuration[i]} is outside the expected range {turnResult.ExpectedTTSAudioResponseDuration[i]}+/-{margin}"); durationMatch = false; turnResult.TTSAudioResponseDurationMatch = false; } } } } } if (!string.IsNullOrWhiteSpace(turnResult.ExpectedResponseLatency) && turnResult.ActualResponseLatency > 0) { int expectedResponseLatency = int.Parse(turnResult.ExpectedResponseLatency.Split(",")[0], CultureInfo.CurrentCulture); if (turnResult.ActualResponseLatency > expectedResponseLatency) { Trace.TraceInformation($"Actual bot response latency {turnResult.ActualResponseLatency} msec exceeds expected latency {expectedResponseLatency} msec"); turnResult.ResponseLatencyMatch = false; } } } turnResult.Pass = turnResult.ResponseMatch && turnResult.UtteranceMatch && turnResult.TTSAudioResponseDurationMatch && turnResult.ResponseLatencyMatch; this.DisplayTestResultMessage(turnResult); return(turnResult.Pass); }
/// <summary> /// Builds the output. /// </summary> /// <param name="turns"> Input Turns.</param> /// <param name="bootstrapMode">Boolean which defines if turn is in bootstrapping mode or not.</param> /// <param name="recognizedText">Recognized text from Speech Recongition.</param> /// <param name="recognizedKeyword">Recogized Keyword from Keyword Recognition.</param> /// <returns>TurnsOutput.</returns> public TurnResult BuildOutput(Turn turns, bool bootstrapMode, string recognizedText, string recognizedKeyword) { TurnResult turnsOutput = new TurnResult(turns) { ActualResponses = new List <Activity>(), ActualTTSAudioResponseDuration = new List <int>(), }; turnsOutput.ActualRecognizedText = recognizedText; if (recognizedKeyword != null) { turnsOutput.KeywordVerified = recognizedKeyword; } foreach (BotReply botReply in this.BotResponses) { turnsOutput.ActualResponses.Add(botReply.Activity); turnsOutput.ActualTTSAudioResponseDuration.Add(botReply.TTSAudioDuration); } if (bootstrapMode) { // In bootstrapping mode, ExpectedResponses field does not exist (or is null), and ExpectedResponseLatency does not exist if (this.BotResponses.Count > 0) { // Report the latency of the last bot response turnsOutput.ActualResponseLatency = this.BotResponses[this.BotResponses.Count - 1].Latency; } } else { // In normal mode, ExpectedResponses exists with one or more activities. ExpectedResponseLatency may or may not exist int activityIndexForLatency = 0; if (string.IsNullOrWhiteSpace(turnsOutput.ExpectedResponseLatency)) { activityIndexForLatency = turns.ExpectedResponses.Count - 1; } else { if (turnsOutput.ExpectedResponseLatency.Split(",").Length == 2) { // The user has specified an expected response latency in the two-integer string format "latency,index". Extract the index // Note: the index has already been verified to be in the range [0, turns.ExpectedResponses.Count - 1] activityIndexForLatency = int.Parse(turnsOutput.ExpectedResponseLatency.Split(",")[1], CultureInfo.CurrentCulture); } else { // The user has specified an expected response latency in the single integer string format, without an index activityIndexForLatency = turns.ExpectedResponses.Count - 1; } } if (activityIndexForLatency < this.BotResponses.Count) { turnsOutput.ActualResponseLatency = this.BotResponses[activityIndexForLatency].Latency; } } return(turnsOutput); }
/// <summary> /// Returns bool values indicating if Actual parameters match expected parameters. /// Parameters include Intents, slots, responses, TTS duration, and latency. /// </summary> /// <param name="turnResult">TurnResult object capturing the responses from the bot for this turn.</param> /// <param name="bootstrapMode"> Boolean which defines if turn is in bootstrapping mode or not.</param> /// <returns>true if the all turn tests pass, false if any of the test failed.</returns> public bool ValidateTurn(TurnResult turnResult, bool bootstrapMode) { turnResult.Pass = true; if (!bootstrapMode) { turnResult.ResponseMatch = true; turnResult.UtteranceMatch = true; turnResult.TTSAudioResponseDurationMatch = true; turnResult.UserPerceivedLatencyMatch = true; int margin = this.appSettings.TTSAudioDurationMargin; if (!string.IsNullOrWhiteSpace(turnResult.WAVFile) && !string.IsNullOrWhiteSpace(turnResult.Utterance)) { string normalizedActualRecognizedText = GetNormalizedText(turnResult.ActualRecognizedText); string normalizedExpectedRecognizedText = GetNormalizedText(turnResult.Utterance); if (!normalizedExpectedRecognizedText.Equals(normalizedActualRecognizedText, StringComparison.OrdinalIgnoreCase)) { Trace.TraceInformation($"Recognized text \"{turnResult.ActualRecognizedText}\" does not match \"{turnResult.Utterance}\""); turnResult.UtteranceMatch = false; } } turnResult.ResponseMatch = DialogResult.ActivityListsMatch(turnResult.ExpectedResponses, turnResult.ActualResponses); if (turnResult.ResponseMatch && turnResult.ExpectedTTSAudioResponseDurations != null && turnResult.ExpectedTTSAudioResponseDurations.Count != 0 && turnResult.ActualTTSAudioResponseDurations != null && turnResult.ActualTTSAudioResponseDurations.Count != 0) { if (turnResult.ExpectedTTSAudioResponseDurations.Count > turnResult.ActualTTSAudioResponseDurations.Count) { turnResult.TTSAudioResponseDurationMatch = false; } else { for (int i = 0; i < turnResult.ExpectedTTSAudioResponseDurations.Count; i++) { int expectedTTSAudioResponseDuration = GetCorrespondingExpectedTTSAudioResponseDuration(turnResult.ExpectedResponses[i], turnResult.ActualResponses[i], turnResult.ExpectedTTSAudioResponseDurations[i]); int actualTTSAudioResponseDuration = turnResult.ActualTTSAudioResponseDurations[i]; if (expectedTTSAudioResponseDuration > 0) { if (actualTTSAudioResponseDuration >= (expectedTTSAudioResponseDuration - margin) && actualTTSAudioResponseDuration <= (expectedTTSAudioResponseDuration + margin)) { turnResult.TTSAudioResponseDurationMatch = true; } else { Trace.TraceInformation($"Actual TTS audio duration {actualTTSAudioResponseDuration} is outside the expected range {expectedTTSAudioResponseDuration}+/-{margin}"); turnResult.TTSAudioResponseDurationMatch = false; } } } } } if (!string.IsNullOrWhiteSpace(turnResult.ExpectedUserPerceivedLatency) && turnResult.ActualUserPerceivedLatency > 0) { int expectedUserPerceivedLatency = int.Parse(turnResult.ExpectedUserPerceivedLatency.Split(",")[0], CultureInfo.CurrentCulture); if (turnResult.ActualUserPerceivedLatency > expectedUserPerceivedLatency) { Trace.TraceInformation($"Actual bot response latency {turnResult.ActualUserPerceivedLatency} msec exceeds expected latency {expectedUserPerceivedLatency} msec"); turnResult.UserPerceivedLatencyMatch = false; } } turnResult.Pass = turnResult.ResponseMatch && turnResult.UtteranceMatch && turnResult.TTSAudioResponseDurationMatch && turnResult.UserPerceivedLatencyMatch; } this.DisplayTestResultMessage(turnResult); return(turnResult.Pass); }