/// <summary>
/// Record the results from a validation test run into this object
/// </summary>
/// <param name="results"></param>
internal void RecordResults(ValidationResults results, CheckServiceOptions options)
{
foreach (var result in results.Results)
{
if ((result.Outcome & ValidationOutcome.Error) > 0)
{
FailureCount++;
}
else if ((result.Outcome & ValidationOutcome.Warning) > 0)
{
if (options.IgnoreWarnings || options.SilenceWarnings)
{
SuccessCount++;
}
else
{
WarningCount++;
}
}
else if ((result.Outcome & ValidationOutcome.Passed) > 0)
{
SuccessCount++;
}
}
}
/// <summary>
/// Execute the provided methods on the given account.
/// </summary>
/// <param name="options"></param>
/// <param name="account"></param>
/// <param name="methods"></param>
/// <param name="docset"></param>
/// <returns>True if the methods all passed, false if there were failures.</returns>
private static async Task <bool> CheckMethodsForAccountAsync(CheckServiceOptions options, IServiceAccount account, MethodDefinition[] methods, DocSet docset)
{
//CheckResults results = new CheckResults();
ConfigureAdditionalHeadersForAccount(options, account);
string testNamePrefix = account.Name.ToLower() + ": ";
FancyConsole.WriteLine(FancyConsole.ConsoleHeaderColor, "Testing with account: {0}", account.Name);
FancyConsole.WriteLine(FancyConsole.ConsoleCodeColor, "Preparing authentication for requests...", account.Name);
try
{
await account.PrepareForRequestAsync();
}
catch (Exception ex)
{
RecordError(ex.Message);
return(false);
}
AuthenicationCredentials credentials = account.CreateCredentials();
int concurrentTasks = options.ParallelTests ? ParallelTaskCount : 1;
CheckResults docSetResults = new CheckResults();
await ForEachAsync(methods, concurrentTasks, async method =>
{
FancyConsole.WriteLine(
FancyConsole.ConsoleCodeColor,
"Running validation for method: {0}",
method.Identifier);
ScenarioDefinition[] scenarios = docset.TestScenarios.ScenariosForMethod(method);
ValidationResults results = await method.ValidateServiceResponseAsync(scenarios, account, credentials);
PrintResultsToConsole(method, account, results, options);
await TestReport.LogMethodTestResults(method, account, results);
docSetResults.RecordResults(results, options);
if (concurrentTasks == 1)
{
AddPause(options);
}
});
if (options.IgnoreWarnings || options.SilenceWarnings)
{
// Remove the warning flag from the outcomes
docSetResults.ConvertWarningsToSuccess();
}
docSetResults.PrintToConsole();
bool hadWarnings = docSetResults.WarningCount > 0;
bool hadErrors = docSetResults.FailureCount > 0;
return(!(hadErrors | hadWarnings));
}
private static void AddPause(CheckServiceOptions options)
{
if (options.PauseBetweenRequests)
{
FancyConsole.WriteLine("Press any key to continue");
Console.ReadKey();
FancyConsole.WriteLine();
}
}
/// <summary>
/// Executes the remote service tests defined in the documentation. This is similar to CheckDocs, expect
/// that the actual requests come from the service instead of the documentation. Prints the errors to
/// the console.
/// </summary>
/// <param name="options"></param>
/// <returns></returns>
private static async Task <bool> CheckServiceAsync(CheckServiceOptions options)
{
// See if we're supposed to run check-service on this branch (assuming we know what branch we're running on)
if (!string.IsNullOrEmpty(options.BranchName))
{
string[] validBranches = null;
if (null != CurrentConfiguration)
{
validBranches = CurrentConfiguration.CheckServiceEnabledBranches;
}
if (null != validBranches && !validBranches.Contains(options.BranchName))
{
RecordWarning("Aborting check-service run. Branch \"{0}\" wasn't in the checkServiceEnabledBranches configuration list.", options.BranchName);
return(true);
}
}
var docset = await GetDocSetAsync(options);
if (null == docset)
{
return(false);
}
FancyConsole.WriteLine();
if (!string.IsNullOrEmpty(options.ODataMetadataLevel))
{
ValidationConfig.ODataMetadataLevel = options.ODataMetadataLevel;
}
if (options.FoundAccounts == null || !options.FoundAccounts.Any())
{
RecordError("No account was found. Cannot connect to the service.");
return(false);
}
var accountsToProcess =
options.FoundAccounts.Where(
x => string.IsNullOrEmpty(options.AccountName)
? x.Enabled
: options.AccountName.Equals(x.Name));
var methods = FindTestMethods(options, docset);
bool allSuccessful = true;
foreach (var account in accountsToProcess)
{
allSuccessful &= await CheckMethodsForAccountAsync(options, account, methods, docset);
}
return(allSuccessful);
}
private static void ConfigureAdditionalHeadersForAccount(CheckServiceOptions options, IServiceAccount account)
{
if (account.AdditionalHeaders != null && account.AdditionalHeaders.Length > 0)
{
// If the account needs additional headers, merge them in.
List <string> headers = new List <string>(account.AdditionalHeaders);
if (options.AdditionalHeaders != null)
{
headers.AddRange(options.AdditionalHeaders.Split('|'));
}
ValidationConfig.AdditionalHttpHeaders = headers.ToArray();
}
else if (options.AdditionalHeaders != null)
{
var headers = options.AdditionalHeaders.Split('|');
ValidationConfig.AdditionalHttpHeaders = headers.ToArray();
}
}
/// <summary>
/// Record the results from a validation test run into this object
/// </summary>
/// <param name="results"></param>
internal void RecordResults(ValidationResults results, CheckServiceOptions options)
{
foreach (var result in results.Results)
{
if ((result.Outcome & ValidationOutcome.Error) > 0)
{
FailureCount++;
}
else if ((result.Outcome & ValidationOutcome.Warning) > 0)
{
if (options.IgnoreWarnings || options.SilenceWarnings)
SuccessCount++;
else
WarningCount++;
}
else if ((result.Outcome & ValidationOutcome.Passed) > 0)
{
SuccessCount++;
}
}
}
/// <summary>
/// Write the results of a test to the output console.
/// </summary>
/// <param name="method"></param>
/// <param name="account"></param>
/// <param name="results"></param>
/// <param name="options"></param>
private static void PrintResultsToConsole(MethodDefinition method, IServiceAccount account, ValidationResults output, CheckServiceOptions options)
{
// Only allow one thread at a time to write to the console so we don't interleave our results.
lock (typeof(Program))
{
FancyConsole.WriteLine(
FancyConsole.ConsoleHeaderColor,
"Testing method {0} with account {1}",
method.Identifier,
account.Name);
foreach (var scenario in output.Results)
{
if (scenario.Errors.Count > 0)
{
FancyConsole.WriteLineIndented(
" ",
FancyConsole.ConsoleSubheaderColor,
"Scenario: {0}",
scenario.Name);
foreach (var message in scenario.Errors)
{
if (options.EnableVerboseOutput || message.IsWarningOrError)
{
FancyConsole.WriteLineIndented(
" ",
FancyConsole.ConsoleDefaultColor,
message.ErrorText);
}
}
if (options.SilenceWarnings && scenario.Outcome == ValidationOutcome.Warning)
{
scenario.Outcome = ValidationOutcome.Passed;
}
FancyConsole.WriteLineIndented(
" ",
scenario.Outcome.ConsoleColor(),
"Scenario finished with outcome: {0}. Duration: {1}",
scenario.Outcome,
scenario.Duration);
}
}
FancyConsole.WriteLineIndented(
" ",
output.OverallOutcome.ConsoleColor(),
"Method testing finished with overall outcome: {0}",
output.OverallOutcome);
FancyConsole.WriteLine();
}
}