Beispiel #1
0
        /// <summary>
        /// Performs an internal consistency check on the methods (requests/responses) in the documentation.
        /// Prints the results of the tests to the console.
        /// </summary>
        /// <param name="options"></param>
        /// <param name="docset"></param>
        /// <returns></returns>
        private static async Task <CheckResults> CheckMethodsAsync(BasicCheckOptions options, DocSet docset)
        {
            MethodDefinition[] methods = FindTestMethods(options, docset);
            CheckResults       results = new CheckResults();

            foreach (var method in methods)
            {
                var testName = "check-method-syntax: " + method.Identifier;
                TestReport.StartTest(testName, method.SourceFile.DisplayName);

                if (string.IsNullOrEmpty(method.ExpectedResponse))
                {
                    await TestReport.FinishTestAsync(testName, TestOutcome.Failed, "Null response where one was expected.");

                    results.FailureCount++;
                    continue;
                }

                var parser           = new HttpParser();
                var expectedResponse = parser.ParseHttpResponse(method.ExpectedResponse);

                ValidationError[] errors;
                method.ValidateResponse(expectedResponse, null, null, out errors);

                await WriteOutErrorsAndFinishTestAsync(errors, options.SilenceWarnings, "   ", "No errors.", false, testName, "Warnings detected", "Errors detected");

                results.IncrementResultCount(errors);
            }

            return(results);
        }
Beispiel #2
0
        public static CheckResults operator +(CheckResults c1, CheckResults c2)
        {
            var result = new CheckResults
            {
                FailureCount = c1.FailureCount + c2.FailureCount,
                WarningCount = c1.WarningCount + c2.WarningCount,
                SuccessCount = c1.SuccessCount + c2.SuccessCount
            };

            var allErrors = new List <ValidationError>();

            if (null != c1.Errors)
            {
                allErrors.AddRange(c1.Errors);
            }
            if (null != c2.Errors)
            {
                allErrors.AddRange(c2.Errors);
            }

            if (allErrors.Count > 0)
            {
                result.Errors = allErrors;
            }

            return(result);
        }
Beispiel #3
0
        /// <summary>
        /// Perform internal consistency checks on the documentation, including verify that
        /// code blocks have proper formatting, that resources are used properly, and that expected
        /// responses and examples conform to the resource definitions.
        /// </summary>
        /// <param name="options"></param>
        /// <param name="docs"></param>
        /// <returns></returns>
        private static async Task <bool> CheckDocsAsync(BasicCheckOptions options, DocSet docs = null)
        {
            var docset = docs ?? await GetDocSetAsync(options);

            if (null == docset)
            {
                return(false);
            }

            FancyConsole.WriteLine();

            var resultMethods = await CheckMethodsAsync(options, docset);

            CheckResults resultExamples = new CheckResults();

            if (string.IsNullOrEmpty(options.MethodName))
            {
                resultExamples = await CheckExamplesAsync(options, docset);
            }

            var combinedResults = resultMethods + resultExamples;

            if (options.IgnoreWarnings)
            {
                combinedResults.ConvertWarningsToSuccess();
            }

            combinedResults.PrintToConsole();

            return(combinedResults.FailureCount == 0);
        }
Beispiel #4
0
        /// <summary>
        /// Validate that the CSDL metadata defined for a service matches the documentation.
        /// </summary>
        /// <param name="options"></param>
        /// <returns></returns>
        private static async Task <bool> CheckServiceMetadataAsync(CheckMetadataOptions options)
        {
            List <Schema> schemas = await TryGetMetadataSchemasAsync(options);

            if (null == schemas)
            {
                return(false);
            }

            FancyConsole.WriteLine(FancyConsole.ConsoleSuccessColor, "  found {0} schema definitions: {1}", schemas.Count, (from s in schemas select s.Namespace).ComponentsJoinedByString(", "));

            var docSet = await GetDocSetAsync(options);

            if (null == docSet)
            {
                return(false);
            }

            const string testname = "validate-service-metadata";

            TestReport.StartTest(testname);

            List <ResourceDefinition> foundResources = ODataParser.GenerateResourcesFromSchemas(schemas);
            CheckResults results = new CheckResults();

            List <ValidationError> collectedErrors = new List <ValidationError>();

            foreach (var resource in foundResources)
            {
                FancyConsole.WriteLine();
                FancyConsole.Write(FancyConsole.ConsoleHeaderColor, "Checking resource: {0}...", resource.Metadata.ResourceType);

                FancyConsole.VerboseWriteLine();
                FancyConsole.VerboseWriteLine(resource.JsonExample);
                FancyConsole.VerboseWriteLine();

                // Verify that this resource matches the documentation
                ValidationError[] errors;
                docSet.ResourceCollection.ValidateJsonExample(resource.Metadata, resource.JsonExample, out errors, new ValidationOptions {
                    RelaxedStringValidation = true
                });
                results.IncrementResultCount(errors);

                collectedErrors.AddRange(errors);

                await WriteOutErrorsAndFinishTestAsync(errors, options.SilenceWarnings, successMessage : " no errors.");
            }

            if (options.IgnoreWarnings)
            {
                results.ConvertWarningsToSuccess();
            }

            var output = (from e in collectedErrors select e.ErrorText).ComponentsJoinedByString("\r\n");

            await TestReport.FinishTestAsync(testname, results.WereFailures?TestOutcome.Failed : TestOutcome.Passed, stdOut : output);

            results.PrintToConsole();
            return(!results.WereFailures);
        }
Beispiel #5
0
        /// <summary>
        /// Execute the provided methods on the given account.
        /// </summary>
        /// <param name="options"></param>
        /// <param name="account"></param>
        /// <param name="methods"></param>
        /// <param name="docset"></param>
        /// <returns>True if the methods all passed, false if there were failures.</returns>
        private static async Task <bool> CheckMethodsForAccountAsync(CheckServiceOptions options, IServiceAccount account, MethodDefinition[] methods, DocSet docset)
        {
            //CheckResults results = new CheckResults();

            ConfigureAdditionalHeadersForAccount(options, account);

            string testNamePrefix = account.Name.ToLower() + ": ";

            FancyConsole.WriteLine(FancyConsole.ConsoleHeaderColor, "Testing with account: {0}", account.Name);

            FancyConsole.WriteLine(FancyConsole.ConsoleCodeColor, "Preparing authentication for requests...", account.Name);
            try
            {
                await account.PrepareForRequestAsync();
            }
            catch (Exception ex)
            {
                RecordError(ex.Message);
                return(false);
            }

            AuthenicationCredentials credentials = account.CreateCredentials();
            int concurrentTasks = options.ParallelTests ? ParallelTaskCount : 1;

            CheckResults docSetResults = new CheckResults();

            await ForEachAsync(methods, concurrentTasks, async method =>
            {
                FancyConsole.WriteLine(
                    FancyConsole.ConsoleCodeColor,
                    "Running validation for method: {0}",
                    method.Identifier);
                ScenarioDefinition[] scenarios = docset.TestScenarios.ScenariosForMethod(method);
                ValidationResults results      = await method.ValidateServiceResponseAsync(scenarios, account, credentials);

                PrintResultsToConsole(method, account, results, options);
                await TestReport.LogMethodTestResults(method, account, results);
                docSetResults.RecordResults(results, options);

                if (concurrentTasks == 1)
                {
                    AddPause(options);
                }
            });

            if (options.IgnoreWarnings || options.SilenceWarnings)
            {
                // Remove the warning flag from the outcomes
                docSetResults.ConvertWarningsToSuccess();
            }

            docSetResults.PrintToConsole();

            bool hadWarnings = docSetResults.WarningCount > 0;
            bool hadErrors   = docSetResults.FailureCount > 0;

            return(!(hadErrors | hadWarnings));
        }
        public static CheckResults operator +(CheckResults c1, CheckResults c2)
        {
            var result = new CheckResults
            {
                FailureCount = c1.FailureCount + c2.FailureCount,
                WarningCount = c1.WarningCount + c2.WarningCount,
                SuccessCount = c1.SuccessCount + c2.SuccessCount
            };

            var allErrors = new List<ValidationError>();
            if (null != c1.Errors)
                allErrors.AddRange(c1.Errors);
            if (null != c2.Errors)
                allErrors.AddRange(c2.Errors);

            if (allErrors.Count > 0)
                result.Errors = allErrors;

            return result;
        }
Beispiel #7
0
        /// <summary>
        /// Perform an internal consistency check on the examples defined in the documentation. Prints
        /// the results of the tests to the console.
        /// </summary>
        /// <param name="options"></param>
        /// <param name="docset"></param>
        /// <returns></returns>
        private static async Task <CheckResults> CheckExamplesAsync(BasicCheckOptions options, DocSet docset)
        {
            var results = new CheckResults();

            foreach (var doc in docset.Files)
            {
                if (doc.Examples.Length == 0)
                {
                    continue;
                }

                FancyConsole.WriteLine(FancyConsole.ConsoleHeaderColor, "Checking examples in \"{0}\"...", doc.DisplayName);

                foreach (var example in doc.Examples)
                {
                    if (example.Metadata == null)
                    {
                        continue;
                    }
                    if (example.Language != CodeLanguage.Json)
                    {
                        continue;
                    }

                    var testName = string.Format("check-example: {0}", example.Metadata.MethodName, example.Metadata.ResourceType);
                    TestReport.StartTest(testName, doc.DisplayName);

                    ValidationError[] errors;
                    docset.ResourceCollection.ValidateJsonExample(example.Metadata, example.SourceExample, out errors);

                    await WriteOutErrorsAndFinishTestAsync(errors, options.SilenceWarnings, "   ", "No errors.", false, testName, "Warnings detected", "Errors detected");

                    results.IncrementResultCount(errors);
                }
            }

            return(results);
        }