/// <summary>
/// Performs an internal consistency check on the methods (requests/responses) in the documentation.
/// Prints the results of the tests to the console.
/// </summary>
/// <param name="options"></param>
/// <param name="docset"></param>
/// <returns></returns>
private static async Task <CheckResults> CheckMethodsAsync(BasicCheckOptions options, DocSet docset)
{
MethodDefinition[] methods = FindTestMethods(options, docset);
CheckResults results = new CheckResults();
foreach (var method in methods)
{
var testName = "check-method-syntax: " + method.Identifier;
TestReport.StartTest(testName, method.SourceFile.DisplayName);
if (string.IsNullOrEmpty(method.ExpectedResponse))
{
await TestReport.FinishTestAsync(testName, TestOutcome.Failed, "Null response where one was expected.");
results.FailureCount++;
continue;
}
var parser = new HttpParser();
var expectedResponse = parser.ParseHttpResponse(method.ExpectedResponse);
ValidationError[] errors;
method.ValidateResponse(expectedResponse, null, null, out errors);
await WriteOutErrorsAndFinishTestAsync(errors, options.SilenceWarnings, " ", "No errors.", false, testName, "Warnings detected", "Errors detected");
results.IncrementResultCount(errors);
}
return(results);
}
public static CheckResults operator +(CheckResults c1, CheckResults c2)
{
var result = new CheckResults
{
FailureCount = c1.FailureCount + c2.FailureCount,
WarningCount = c1.WarningCount + c2.WarningCount,
SuccessCount = c1.SuccessCount + c2.SuccessCount
};
var allErrors = new List <ValidationError>();
if (null != c1.Errors)
{
allErrors.AddRange(c1.Errors);
}
if (null != c2.Errors)
{
allErrors.AddRange(c2.Errors);
}
if (allErrors.Count > 0)
{
result.Errors = allErrors;
}
return(result);
}
/// <summary>
/// Perform internal consistency checks on the documentation, including verify that
/// code blocks have proper formatting, that resources are used properly, and that expected
/// responses and examples conform to the resource definitions.
/// </summary>
/// <param name="options"></param>
/// <param name="docs"></param>
/// <returns></returns>
private static async Task <bool> CheckDocsAsync(BasicCheckOptions options, DocSet docs = null)
{
var docset = docs ?? await GetDocSetAsync(options);
if (null == docset)
{
return(false);
}
FancyConsole.WriteLine();
var resultMethods = await CheckMethodsAsync(options, docset);
CheckResults resultExamples = new CheckResults();
if (string.IsNullOrEmpty(options.MethodName))
{
resultExamples = await CheckExamplesAsync(options, docset);
}
var combinedResults = resultMethods + resultExamples;
if (options.IgnoreWarnings)
{
combinedResults.ConvertWarningsToSuccess();
}
combinedResults.PrintToConsole();
return(combinedResults.FailureCount == 0);
}
/// <summary>
/// Validate that the CSDL metadata defined for a service matches the documentation.
/// </summary>
/// <param name="options"></param>
/// <returns></returns>
private static async Task <bool> CheckServiceMetadataAsync(CheckMetadataOptions options)
{
List <Schema> schemas = await TryGetMetadataSchemasAsync(options);
if (null == schemas)
{
return(false);
}
FancyConsole.WriteLine(FancyConsole.ConsoleSuccessColor, " found {0} schema definitions: {1}", schemas.Count, (from s in schemas select s.Namespace).ComponentsJoinedByString(", "));
var docSet = await GetDocSetAsync(options);
if (null == docSet)
{
return(false);
}
const string testname = "validate-service-metadata";
TestReport.StartTest(testname);
List <ResourceDefinition> foundResources = ODataParser.GenerateResourcesFromSchemas(schemas);
CheckResults results = new CheckResults();
List <ValidationError> collectedErrors = new List <ValidationError>();
foreach (var resource in foundResources)
{
FancyConsole.WriteLine();
FancyConsole.Write(FancyConsole.ConsoleHeaderColor, "Checking resource: {0}...", resource.Metadata.ResourceType);
FancyConsole.VerboseWriteLine();
FancyConsole.VerboseWriteLine(resource.JsonExample);
FancyConsole.VerboseWriteLine();
// Verify that this resource matches the documentation
ValidationError[] errors;
docSet.ResourceCollection.ValidateJsonExample(resource.Metadata, resource.JsonExample, out errors, new ValidationOptions {
RelaxedStringValidation = true
});
results.IncrementResultCount(errors);
collectedErrors.AddRange(errors);
await WriteOutErrorsAndFinishTestAsync(errors, options.SilenceWarnings, successMessage : " no errors.");
}
if (options.IgnoreWarnings)
{
results.ConvertWarningsToSuccess();
}
var output = (from e in collectedErrors select e.ErrorText).ComponentsJoinedByString("\r\n");
await TestReport.FinishTestAsync(testname, results.WereFailures?TestOutcome.Failed : TestOutcome.Passed, stdOut : output);
results.PrintToConsole();
return(!results.WereFailures);
}
/// <summary>
/// Execute the provided methods on the given account.
/// </summary>
/// <param name="options"></param>
/// <param name="account"></param>
/// <param name="methods"></param>
/// <param name="docset"></param>
/// <returns>True if the methods all passed, false if there were failures.</returns>
private static async Task <bool> CheckMethodsForAccountAsync(CheckServiceOptions options, IServiceAccount account, MethodDefinition[] methods, DocSet docset)
{
//CheckResults results = new CheckResults();
ConfigureAdditionalHeadersForAccount(options, account);
string testNamePrefix = account.Name.ToLower() + ": ";
FancyConsole.WriteLine(FancyConsole.ConsoleHeaderColor, "Testing with account: {0}", account.Name);
FancyConsole.WriteLine(FancyConsole.ConsoleCodeColor, "Preparing authentication for requests...", account.Name);
try
{
await account.PrepareForRequestAsync();
}
catch (Exception ex)
{
RecordError(ex.Message);
return(false);
}
AuthenicationCredentials credentials = account.CreateCredentials();
int concurrentTasks = options.ParallelTests ? ParallelTaskCount : 1;
CheckResults docSetResults = new CheckResults();
await ForEachAsync(methods, concurrentTasks, async method =>
{
FancyConsole.WriteLine(
FancyConsole.ConsoleCodeColor,
"Running validation for method: {0}",
method.Identifier);
ScenarioDefinition[] scenarios = docset.TestScenarios.ScenariosForMethod(method);
ValidationResults results = await method.ValidateServiceResponseAsync(scenarios, account, credentials);
PrintResultsToConsole(method, account, results, options);
await TestReport.LogMethodTestResults(method, account, results);
docSetResults.RecordResults(results, options);
if (concurrentTasks == 1)
{
AddPause(options);
}
});
if (options.IgnoreWarnings || options.SilenceWarnings)
{
// Remove the warning flag from the outcomes
docSetResults.ConvertWarningsToSuccess();
}
docSetResults.PrintToConsole();
bool hadWarnings = docSetResults.WarningCount > 0;
bool hadErrors = docSetResults.FailureCount > 0;
return(!(hadErrors | hadWarnings));
}
public static CheckResults operator +(CheckResults c1, CheckResults c2)
{
var result = new CheckResults
{
FailureCount = c1.FailureCount + c2.FailureCount,
WarningCount = c1.WarningCount + c2.WarningCount,
SuccessCount = c1.SuccessCount + c2.SuccessCount
};
var allErrors = new List<ValidationError>();
if (null != c1.Errors)
allErrors.AddRange(c1.Errors);
if (null != c2.Errors)
allErrors.AddRange(c2.Errors);
if (allErrors.Count > 0)
result.Errors = allErrors;
return result;
}
/// <summary>
/// Perform an internal consistency check on the examples defined in the documentation. Prints
/// the results of the tests to the console.
/// </summary>
/// <param name="options"></param>
/// <param name="docset"></param>
/// <returns></returns>
private static async Task <CheckResults> CheckExamplesAsync(BasicCheckOptions options, DocSet docset)
{
var results = new CheckResults();
foreach (var doc in docset.Files)
{
if (doc.Examples.Length == 0)
{
continue;
}
FancyConsole.WriteLine(FancyConsole.ConsoleHeaderColor, "Checking examples in \"{0}\"...", doc.DisplayName);
foreach (var example in doc.Examples)
{
if (example.Metadata == null)
{
continue;
}
if (example.Language != CodeLanguage.Json)
{
continue;
}
var testName = string.Format("check-example: {0}", example.Metadata.MethodName, example.Metadata.ResourceType);
TestReport.StartTest(testName, doc.DisplayName);
ValidationError[] errors;
docset.ResourceCollection.ValidateJsonExample(example.Metadata, example.SourceExample, out errors);
await WriteOutErrorsAndFinishTestAsync(errors, options.SilenceWarnings, " ", "No errors.", false, testName, "Warnings detected", "Errors detected");
results.IncrementResultCount(errors);
}
}
return(results);
}