/// <summary> /// Performs an internal consistency check on the methods (requests/responses) in the documentation. /// Prints the results of the tests to the console. /// </summary> /// <param name="options"></param> /// <param name="docset"></param> /// <returns></returns> private static async Task <CheckResults> CheckMethodsAsync(BasicCheckOptions options, DocSet docset) { MethodDefinition[] methods = FindTestMethods(options, docset); CheckResults results = new CheckResults(); foreach (var method in methods) { var testName = "check-method-syntax: " + method.Identifier; TestReport.StartTest(testName, method.SourceFile.DisplayName); if (string.IsNullOrEmpty(method.ExpectedResponse)) { await TestReport.FinishTestAsync(testName, TestOutcome.Failed, "Null response where one was expected."); results.FailureCount++; continue; } var parser = new HttpParser(); var expectedResponse = parser.ParseHttpResponse(method.ExpectedResponse); ValidationError[] errors; method.ValidateResponse(expectedResponse, null, null, out errors); await WriteOutErrorsAndFinishTestAsync(errors, options.SilenceWarnings, " ", "No errors.", false, testName, "Warnings detected", "Errors detected"); results.IncrementResultCount(errors); } return(results); }
public static CheckResults operator +(CheckResults c1, CheckResults c2) { var result = new CheckResults { FailureCount = c1.FailureCount + c2.FailureCount, WarningCount = c1.WarningCount + c2.WarningCount, SuccessCount = c1.SuccessCount + c2.SuccessCount }; var allErrors = new List <ValidationError>(); if (null != c1.Errors) { allErrors.AddRange(c1.Errors); } if (null != c2.Errors) { allErrors.AddRange(c2.Errors); } if (allErrors.Count > 0) { result.Errors = allErrors; } return(result); }
/// <summary> /// Perform internal consistency checks on the documentation, including verify that /// code blocks have proper formatting, that resources are used properly, and that expected /// responses and examples conform to the resource definitions. /// </summary> /// <param name="options"></param> /// <param name="docs"></param> /// <returns></returns> private static async Task <bool> CheckDocsAsync(BasicCheckOptions options, DocSet docs = null) { var docset = docs ?? await GetDocSetAsync(options); if (null == docset) { return(false); } FancyConsole.WriteLine(); var resultMethods = await CheckMethodsAsync(options, docset); CheckResults resultExamples = new CheckResults(); if (string.IsNullOrEmpty(options.MethodName)) { resultExamples = await CheckExamplesAsync(options, docset); } var combinedResults = resultMethods + resultExamples; if (options.IgnoreWarnings) { combinedResults.ConvertWarningsToSuccess(); } combinedResults.PrintToConsole(); return(combinedResults.FailureCount == 0); }
/// <summary> /// Validate that the CSDL metadata defined for a service matches the documentation. /// </summary> /// <param name="options"></param> /// <returns></returns> private static async Task <bool> CheckServiceMetadataAsync(CheckMetadataOptions options) { List <Schema> schemas = await TryGetMetadataSchemasAsync(options); if (null == schemas) { return(false); } FancyConsole.WriteLine(FancyConsole.ConsoleSuccessColor, " found {0} schema definitions: {1}", schemas.Count, (from s in schemas select s.Namespace).ComponentsJoinedByString(", ")); var docSet = await GetDocSetAsync(options); if (null == docSet) { return(false); } const string testname = "validate-service-metadata"; TestReport.StartTest(testname); List <ResourceDefinition> foundResources = ODataParser.GenerateResourcesFromSchemas(schemas); CheckResults results = new CheckResults(); List <ValidationError> collectedErrors = new List <ValidationError>(); foreach (var resource in foundResources) { FancyConsole.WriteLine(); FancyConsole.Write(FancyConsole.ConsoleHeaderColor, "Checking resource: {0}...", resource.Metadata.ResourceType); FancyConsole.VerboseWriteLine(); FancyConsole.VerboseWriteLine(resource.JsonExample); FancyConsole.VerboseWriteLine(); // Verify that this resource matches the documentation ValidationError[] errors; docSet.ResourceCollection.ValidateJsonExample(resource.Metadata, resource.JsonExample, out errors, new ValidationOptions { RelaxedStringValidation = true }); results.IncrementResultCount(errors); collectedErrors.AddRange(errors); await WriteOutErrorsAndFinishTestAsync(errors, options.SilenceWarnings, successMessage : " no errors."); } if (options.IgnoreWarnings) { results.ConvertWarningsToSuccess(); } var output = (from e in collectedErrors select e.ErrorText).ComponentsJoinedByString("\r\n"); await TestReport.FinishTestAsync(testname, results.WereFailures?TestOutcome.Failed : TestOutcome.Passed, stdOut : output); results.PrintToConsole(); return(!results.WereFailures); }
/// <summary> /// Execute the provided methods on the given account. /// </summary> /// <param name="options"></param> /// <param name="account"></param> /// <param name="methods"></param> /// <param name="docset"></param> /// <returns>True if the methods all passed, false if there were failures.</returns> private static async Task <bool> CheckMethodsForAccountAsync(CheckServiceOptions options, IServiceAccount account, MethodDefinition[] methods, DocSet docset) { //CheckResults results = new CheckResults(); ConfigureAdditionalHeadersForAccount(options, account); string testNamePrefix = account.Name.ToLower() + ": "; FancyConsole.WriteLine(FancyConsole.ConsoleHeaderColor, "Testing with account: {0}", account.Name); FancyConsole.WriteLine(FancyConsole.ConsoleCodeColor, "Preparing authentication for requests...", account.Name); try { await account.PrepareForRequestAsync(); } catch (Exception ex) { RecordError(ex.Message); return(false); } AuthenicationCredentials credentials = account.CreateCredentials(); int concurrentTasks = options.ParallelTests ? ParallelTaskCount : 1; CheckResults docSetResults = new CheckResults(); await ForEachAsync(methods, concurrentTasks, async method => { FancyConsole.WriteLine( FancyConsole.ConsoleCodeColor, "Running validation for method: {0}", method.Identifier); ScenarioDefinition[] scenarios = docset.TestScenarios.ScenariosForMethod(method); ValidationResults results = await method.ValidateServiceResponseAsync(scenarios, account, credentials); PrintResultsToConsole(method, account, results, options); await TestReport.LogMethodTestResults(method, account, results); docSetResults.RecordResults(results, options); if (concurrentTasks == 1) { AddPause(options); } }); if (options.IgnoreWarnings || options.SilenceWarnings) { // Remove the warning flag from the outcomes docSetResults.ConvertWarningsToSuccess(); } docSetResults.PrintToConsole(); bool hadWarnings = docSetResults.WarningCount > 0; bool hadErrors = docSetResults.FailureCount > 0; return(!(hadErrors | hadWarnings)); }
public static CheckResults operator +(CheckResults c1, CheckResults c2) { var result = new CheckResults { FailureCount = c1.FailureCount + c2.FailureCount, WarningCount = c1.WarningCount + c2.WarningCount, SuccessCount = c1.SuccessCount + c2.SuccessCount }; var allErrors = new List<ValidationError>(); if (null != c1.Errors) allErrors.AddRange(c1.Errors); if (null != c2.Errors) allErrors.AddRange(c2.Errors); if (allErrors.Count > 0) result.Errors = allErrors; return result; }
/// <summary> /// Perform an internal consistency check on the examples defined in the documentation. Prints /// the results of the tests to the console. /// </summary> /// <param name="options"></param> /// <param name="docset"></param> /// <returns></returns> private static async Task <CheckResults> CheckExamplesAsync(BasicCheckOptions options, DocSet docset) { var results = new CheckResults(); foreach (var doc in docset.Files) { if (doc.Examples.Length == 0) { continue; } FancyConsole.WriteLine(FancyConsole.ConsoleHeaderColor, "Checking examples in \"{0}\"...", doc.DisplayName); foreach (var example in doc.Examples) { if (example.Metadata == null) { continue; } if (example.Language != CodeLanguage.Json) { continue; } var testName = string.Format("check-example: {0}", example.Metadata.MethodName, example.Metadata.ResourceType); TestReport.StartTest(testName, doc.DisplayName); ValidationError[] errors; docset.ResourceCollection.ValidateJsonExample(example.Metadata, example.SourceExample, out errors); await WriteOutErrorsAndFinishTestAsync(errors, options.SilenceWarnings, " ", "No errors.", false, testName, "Warnings detected", "Errors detected"); results.IncrementResultCount(errors); } } return(results); }