private static async Task <bool> ProcessDialog( List <Dialog> fileContents, BotConnector botConnector, AppSettings appSettings, bool isFirstDialog, TestSettings tests, bool connectionEstablished, string testName, List <DialogReport> dialogReports, bool testPass, List <DialogResult> dialogResults, bool sendFirst) { foreach (Dialog dialog in fileContents) { Trace.IndentLevel = 1; if (dialog.Skip) { Trace.TraceInformation($"Skipping DialogID {dialog.DialogID}"); continue; } else { Trace.TraceInformation($"[{DateTime.Now.ToString("hh:mm:ss.fff", CultureInfo.CurrentCulture)}] Running DialogId {dialog.DialogID}, description \"{dialog.Description}\""); } // Capture and compute the output for this dialog in this variable. DialogResult dialogResult = new DialogResult(appSettings, dialog.DialogID, dialog.Description); // Capture outputs of all turns in this dialog in this list. List <TurnResult> turnResults = new List <TurnResult>(); // Keep track of turn pass/fail : per turn. List <bool> turnPassResults = new List <bool>(); if (isFirstDialog || tests.SingleConnection == false) { // Always establish a connection with the bot for the first dialog in the test file. // If SingleConnection is false, it also means we need to re-establish the connection before each of the dialogs in the test file. if (botConnector != null) { await botConnector.Disconnect().ConfigureAwait(false); botConnector.Dispose(); } connectionEstablished = true; botConnector = new BotConnector(); botConnector.InitConnector(appSettings); await botConnector.Connect().ConfigureAwait(false); } isFirstDialog = false; foreach (Turn turn in dialog.Turns) { // Application crashes in a multi-turn dialog with Keyword in each Turn // Crash occurs when calling StartKeywordRecognitionAsync after calling StopKeywordRecognitionAsync in the previous Turn. // In order to avoid this crash, only have Keyword in Turn 0 of a multi-turn Keyword containing Dialog. // This is being investigated. // MS-Internal bug number: 2300634. // https://msasg.visualstudio.com/Skyman/_workitems/edit/2300634/ if (turn.Keyword) { await botConnector.StartKeywordRecognitionAsync().ConfigureAwait(false); } Trace.IndentLevel = 2; Trace.TraceInformation($"[{DateTime.Now.ToString("hh:mm:ss.fff", CultureInfo.CurrentCulture)}] Running Turn {turn.TurnID}"); Trace.IndentLevel = 3; if (turn.Sleep > 0) { Trace.TraceInformation($"Sleeping for {turn.Sleep} msec"); System.Threading.Thread.Sleep(turn.Sleep); } int responseCount = 0; bool bootstrapMode = true; if (turn.ExpectedResponses != null && turn.ExpectedResponses.Count != 0) { responseCount = turn.ExpectedResponses.Count; bootstrapMode = false; } botConnector.SetInputValues(testName, dialog.DialogID, turn.TurnID, responseCount, tests.IgnoreActivities, turn.Keyword); if (tests.WavAndUtterancePairs && !string.IsNullOrWhiteSpace(turn.WAVFile) && !string.IsNullOrWhiteSpace(turn.Utterance)) { // Send up WAV File if present if (!string.IsNullOrEmpty(turn.WAVFile) && sendFirst) { botConnector.SendAudio(turn.WAVFile); } // Send up Utterance if present else if (!string.IsNullOrEmpty(turn.Utterance) && !sendFirst) { botConnector = await botConnector.Send(turn.Utterance).ConfigureAwait(false); } } // WavAndUtterancePair is false send either wavfile or utterance if present. else if (!tests.WavAndUtterancePairs) { // Send up WAV File if present if (!string.IsNullOrEmpty(turn.WAVFile)) { botConnector.SendAudio(turn.WAVFile); } // Send up Utterance if present else if (!string.IsNullOrEmpty(turn.Utterance)) { botConnector = await botConnector.Send(turn.Utterance).ConfigureAwait(false); } } // Send up activity if configured else if (!string.IsNullOrEmpty(turn.Activity)) { botConnector = await botConnector.SendActivity(turn.Activity).ConfigureAwait(false); } // All bot reply activities are captured in this variable. dialogResult.BotResponses = botConnector.WaitAndProcessBotReplies(bootstrapMode); // Capture the result of this turn in this variable and validate the turn. TurnResult turnResult = dialogResult.BuildOutput(turn, bootstrapMode, botConnector.RecognizedText, botConnector.RecognizedKeyword); if (!dialogResult.ValidateTurn(turnResult, bootstrapMode)) { testPass = false; } // Add the turn result to the list of turn results. turnResults.Add(turnResult); // Add the turn completion status to the list of turn completions. turnPassResults.Add(turnResult.Pass); if (turn.Keyword) { await botConnector.StopKeywordRecognitionAsync().ConfigureAwait(false); } } // End of turns loop dialogResult.Turns = turnResults; dialogResults.Add(dialogResult); DialogReport dialogReport = new DialogReport(dialogResult.DialogID, dialog.Description, turnPassResults); dialogReports.Add(dialogReport); turnPassResults = new List <bool>(); Trace.IndentLevel = 1; if (dialogReport.DialogPass) { Trace.TraceInformation($"DialogId {dialog.DialogID} passed"); #if USE_ARIA_LOGGING AriaLogger.Log(AriaLogger.EventNameDialogSucceeded, dialog.DialogID, dialog.Description); #endif } else { Trace.TraceInformation($"DialogId {dialog.DialogID} failed"); #if USE_ARIA_LOGGING AriaLogger.Log(AriaLogger.EventNameDialogFailed, dialog.DialogID, dialog.Description); #endif } } // End of dialog loop return(testPass); }
private static async Task ProcessDialogAndGenerateReport(string outputType, string outputPath, bool testPass, BotConnector botConnector, List <DialogReport> dialogReports, bool connectionEstablished, List <DialogResult> dialogResults, List <Dialog> fileContents, bool isFirstDialog, TestSettings tests, string testName, bool sendFirst, string inputFileName, List <TestReport> allInputFilesTestReport, string outputFileName, AppSettings appSettings) { outputPath = Path.Combine(appSettings.OutputFolder, testName + outputType); testName = Path.GetFileNameWithoutExtension(outputPath); DirectoryInfo outputDirectory = Directory.CreateDirectory(outputPath); outputFileName = Path.Combine(outputDirectory.FullName, testName + ".json"); testPass = await ProcessDialog(fileContents, botConnector, appSettings, isFirstDialog, tests, connectionEstablished, testName, dialogReports, testPass, dialogResults, sendFirst).ConfigureAwait(false); await ProcessTestReport(inputFileName, dialogReports, allInputFilesTestReport, botConnector, testPass, dialogResults, outputFileName, connectionEstablished, appSettings).ConfigureAwait(false); }