/// <summary> /// Record the results from a validation test run into this object /// </summary> /// <param name="results"></param> internal void RecordResults(ValidationResults results, CheckServiceOptions options) { foreach (var result in results.Results) { if ((result.Outcome & ValidationOutcome.Error) > 0) { FailureCount++; } else if ((result.Outcome & ValidationOutcome.Warning) > 0) { if (options.IgnoreWarnings || options.SilenceWarnings) { SuccessCount++; } else { WarningCount++; } } else if ((result.Outcome & ValidationOutcome.Passed) > 0) { SuccessCount++; } } }
/// <summary> /// Execute the provided methods on the given account. /// </summary> /// <param name="options"></param> /// <param name="account"></param> /// <param name="methods"></param> /// <param name="docset"></param> /// <returns>True if the methods all passed, false if there were failures.</returns> private static async Task <bool> CheckMethodsForAccountAsync(CheckServiceOptions options, IServiceAccount account, MethodDefinition[] methods, DocSet docset) { //CheckResults results = new CheckResults(); ConfigureAdditionalHeadersForAccount(options, account); string testNamePrefix = account.Name.ToLower() + ": "; FancyConsole.WriteLine(FancyConsole.ConsoleHeaderColor, "Testing with account: {0}", account.Name); FancyConsole.WriteLine(FancyConsole.ConsoleCodeColor, "Preparing authentication for requests...", account.Name); try { await account.PrepareForRequestAsync(); } catch (Exception ex) { RecordError(ex.Message); return(false); } AuthenicationCredentials credentials = account.CreateCredentials(); int concurrentTasks = options.ParallelTests ? ParallelTaskCount : 1; CheckResults docSetResults = new CheckResults(); await ForEachAsync(methods, concurrentTasks, async method => { FancyConsole.WriteLine( FancyConsole.ConsoleCodeColor, "Running validation for method: {0}", method.Identifier); ScenarioDefinition[] scenarios = docset.TestScenarios.ScenariosForMethod(method); ValidationResults results = await method.ValidateServiceResponseAsync(scenarios, account, credentials); PrintResultsToConsole(method, account, results, options); await TestReport.LogMethodTestResults(method, account, results); docSetResults.RecordResults(results, options); if (concurrentTasks == 1) { AddPause(options); } }); if (options.IgnoreWarnings || options.SilenceWarnings) { // Remove the warning flag from the outcomes docSetResults.ConvertWarningsToSuccess(); } docSetResults.PrintToConsole(); bool hadWarnings = docSetResults.WarningCount > 0; bool hadErrors = docSetResults.FailureCount > 0; return(!(hadErrors | hadWarnings)); }
private static void AddPause(CheckServiceOptions options) { if (options.PauseBetweenRequests) { FancyConsole.WriteLine("Press any key to continue"); Console.ReadKey(); FancyConsole.WriteLine(); } }
/// <summary> /// Executes the remote service tests defined in the documentation. This is similar to CheckDocs, expect /// that the actual requests come from the service instead of the documentation. Prints the errors to /// the console. /// </summary> /// <param name="options"></param> /// <returns></returns> private static async Task <bool> CheckServiceAsync(CheckServiceOptions options) { // See if we're supposed to run check-service on this branch (assuming we know what branch we're running on) if (!string.IsNullOrEmpty(options.BranchName)) { string[] validBranches = null; if (null != CurrentConfiguration) { validBranches = CurrentConfiguration.CheckServiceEnabledBranches; } if (null != validBranches && !validBranches.Contains(options.BranchName)) { RecordWarning("Aborting check-service run. Branch \"{0}\" wasn't in the checkServiceEnabledBranches configuration list.", options.BranchName); return(true); } } var docset = await GetDocSetAsync(options); if (null == docset) { return(false); } FancyConsole.WriteLine(); if (!string.IsNullOrEmpty(options.ODataMetadataLevel)) { ValidationConfig.ODataMetadataLevel = options.ODataMetadataLevel; } if (options.FoundAccounts == null || !options.FoundAccounts.Any()) { RecordError("No account was found. Cannot connect to the service."); return(false); } var accountsToProcess = options.FoundAccounts.Where( x => string.IsNullOrEmpty(options.AccountName) ? x.Enabled : options.AccountName.Equals(x.Name)); var methods = FindTestMethods(options, docset); bool allSuccessful = true; foreach (var account in accountsToProcess) { allSuccessful &= await CheckMethodsForAccountAsync(options, account, methods, docset); } return(allSuccessful); }
private static void ConfigureAdditionalHeadersForAccount(CheckServiceOptions options, IServiceAccount account) { if (account.AdditionalHeaders != null && account.AdditionalHeaders.Length > 0) { // If the account needs additional headers, merge them in. List <string> headers = new List <string>(account.AdditionalHeaders); if (options.AdditionalHeaders != null) { headers.AddRange(options.AdditionalHeaders.Split('|')); } ValidationConfig.AdditionalHttpHeaders = headers.ToArray(); } else if (options.AdditionalHeaders != null) { var headers = options.AdditionalHeaders.Split('|'); ValidationConfig.AdditionalHttpHeaders = headers.ToArray(); } }
/// <summary> /// Record the results from a validation test run into this object /// </summary> /// <param name="results"></param> internal void RecordResults(ValidationResults results, CheckServiceOptions options) { foreach (var result in results.Results) { if ((result.Outcome & ValidationOutcome.Error) > 0) { FailureCount++; } else if ((result.Outcome & ValidationOutcome.Warning) > 0) { if (options.IgnoreWarnings || options.SilenceWarnings) SuccessCount++; else WarningCount++; } else if ((result.Outcome & ValidationOutcome.Passed) > 0) { SuccessCount++; } } }
/// <summary> /// Write the results of a test to the output console. /// </summary> /// <param name="method"></param> /// <param name="account"></param> /// <param name="results"></param> /// <param name="options"></param> private static void PrintResultsToConsole(MethodDefinition method, IServiceAccount account, ValidationResults output, CheckServiceOptions options) { // Only allow one thread at a time to write to the console so we don't interleave our results. lock (typeof(Program)) { FancyConsole.WriteLine( FancyConsole.ConsoleHeaderColor, "Testing method {0} with account {1}", method.Identifier, account.Name); foreach (var scenario in output.Results) { if (scenario.Errors.Count > 0) { FancyConsole.WriteLineIndented( " ", FancyConsole.ConsoleSubheaderColor, "Scenario: {0}", scenario.Name); foreach (var message in scenario.Errors) { if (options.EnableVerboseOutput || message.IsWarningOrError) { FancyConsole.WriteLineIndented( " ", FancyConsole.ConsoleDefaultColor, message.ErrorText); } } if (options.SilenceWarnings && scenario.Outcome == ValidationOutcome.Warning) { scenario.Outcome = ValidationOutcome.Passed; } FancyConsole.WriteLineIndented( " ", scenario.Outcome.ConsoleColor(), "Scenario finished with outcome: {0}. Duration: {1}", scenario.Outcome, scenario.Duration); } } FancyConsole.WriteLineIndented( " ", output.OverallOutcome.ConsoleColor(), "Method testing finished with overall outcome: {0}", output.OverallOutcome); FancyConsole.WriteLine(); } }