/// <summary> /// Performs validation of a method (request/response) with a given service /// target and zero or more test scenarios /// </summary> /// <param name="method"></param> /// <param name="account"></param> /// <param name="credentials"></param> /// <returns></returns> public static async Task<ValidationResults> ValidateServiceResponseAsync( this MethodDefinition method, ScenarioDefinition[] scenarios, IServiceAccount account, ValidationOptions options = null) { if (null == method) throw new ArgumentNullException("method"); if (null == account) throw new ArgumentNullException("account"); ValidationResults results = new ValidationResults(); if (scenarios.Length == 0) { // If no descenarios are defined for this method, add a new default scenario scenarios = new ScenarioDefinition[] { new ScenarioDefinition { Description = "verbatim", Enabled = true, MethodName = method.Identifier, RequiredScopes = method.RequiredScopes } }; // results.AddResult("init", new ValidationMessage(null, "No scenarios were defined for method {0}. Will request verbatim from docs.", method.Identifier), ValidationOutcome.None); } if (scenarios.Any() && !scenarios.Any(x => x.Enabled)) { results.AddResult("init", new ValidationWarning(ValidationErrorCode.AllScenariosDisabled, null, "All scenarios for method {0} were disabled.", method.Identifier), ValidationOutcome.Skipped); return results; } foreach (var scenario in scenarios.Where(x => x.Enabled)) { try { await ValidateMethodWithScenarioAsync(method, scenario, account, results, options); } catch (Exception ex) { results.AddResult( "validation", new ValidationError( ValidationErrorCode.ExceptionWhileValidatingMethod, method.SourceFile.DisplayName, ex.Message)); } } return results; }
private static async Task ValidateMethodWithScenarioAsync( MethodDefinition method, ScenarioDefinition scenario, IServiceAccount account, AuthenicationCredentials credentials, ValidationResults results) { if (null == method) throw new ArgumentNullException("method"); if (null == scenario) throw new ArgumentNullException("scenario"); if (null == account) throw new ArgumentNullException("account"); if (null == credentials) throw new ArgumentNullException("credentials"); if (null == results) throw new ArgumentNullException("results"); var actionName = scenario.Description; // Generate the tested request by "previewing" the request and executing // all test-setup procedures long startTicks = DateTimeOffset.UtcNow.Ticks; var requestPreviewResult = await method.GenerateMethodRequestAsync(scenario, account.BaseUrl, credentials, method.SourceFile.Parent); TimeSpan generateMethodDuration = new TimeSpan(DateTimeOffset.UtcNow.Ticks - startTicks); // Check to see if an error occured building the request, and abort if so. var generatorResults = results[actionName + " [test-setup requests]"]; generatorResults.AddResults(requestPreviewResult.Messages, requestPreviewResult.IsWarningOrError ? ValidationOutcome.Error : ValidationOutcome.Passed); generatorResults.Duration = generateMethodDuration; if (requestPreviewResult.IsWarningOrError) { return; } // We've done all the test-setup work, now we have the real request to make to the service HttpRequest requestPreview = requestPreviewResult.Value; results.AddResult( actionName, new ValidationMessage(null, "Generated Method HTTP Request:\r\n{0}", requestPreview.FullHttpText())); HttpParser parser = new HttpParser(); HttpResponse expectedResponse = null; if (!string.IsNullOrEmpty(method.ExpectedResponse)) { expectedResponse = parser.ParseHttpResponse(method.ExpectedResponse); } // Execute the actual tested method (the result of the method preview call, which made the test-setup requests) startTicks = DateTimeOffset.UtcNow.Ticks; var actualResponse = await requestPreview.GetResponseAsync(account.BaseUrl); TimeSpan actualMethodDuration = new TimeSpan(DateTimeOffset.UtcNow.Ticks - startTicks); var requestResults = results[actionName]; if (actualResponse.RetryCount > 0) { requestResults.AddResults( new ValidationError[] { new ValidationWarning(ValidationErrorCode.RequestWasRetried, null, "HTTP request was retried {0} times.", actualResponse.RetryCount) }); } requestResults.AddResults( new ValidationError[] { new ValidationMessage(null, "HTTP Response:\r\n{0}", actualResponse.FullText(false)) }); requestResults.Duration = actualMethodDuration; // Perform validation on the method's actual response ValidationError[] errors; method.ValidateResponse(actualResponse, expectedResponse, scenario, out errors); requestResults.AddResults(errors); // TODO: If the method is defined as a long running operation, we need to go poll the status // URL to make sure that the operation finished and the response type is valid. if (errors.WereErrors()) results.SetOutcome(actionName, ValidationOutcome.Error); else if (errors.WereWarnings()) results.SetOutcome(actionName, ValidationOutcome.Warning); else results.SetOutcome(actionName, ValidationOutcome.Passed); }
/// <summary> /// Make a call to the HttpRequest and populate the Value property of /// every PlaceholderValue in Values based on the result. /// </summary> /// <param name="storedValues"></param> /// <param name="documents"></param> /// <param name="scenario"></param> /// <param name="account"></param> /// <param name="parentOutputValues"></param> /// <returns></returns> public async Task<ValidationResult<bool>> MakeSetupRequestAsync(Dictionary<string, string> storedValues, DocSet documents, ScenarioDefinition scenario, IServiceAccount account, Dictionary<string, string> parentOutputValues = null) { // Copy the output values from parentOutputValues into our own if (null != parentOutputValues) { foreach (var key in parentOutputValues.Keys) { this.OutputValues.Add(key, parentOutputValues[key]); } } var errors = new List<ValidationError>(); if (!string.IsNullOrEmpty(this.CannedRequestName)) { // We need to make a canned request setup request instead. Look up the canned request and then execute it, returning the results here. var cannedRequest = (from cr in documents.CannedRequests where cr.Name == this.CannedRequestName select cr) .FirstOrDefault(); if (null == cannedRequest) { errors.Add(new ValidationError(ValidationErrorCode.InvalidRequestFormat, null, "Couldn't locate the canned-request named: {0}", this.CannedRequestName)); return new ValidationResult<bool>(false, errors); } // Need to make a copy of the canned request here so that our state doesn't continue to pile up var cannedRequestInstance = cannedRequest.CopyInstance(); return await cannedRequestInstance.MakeSetupRequestAsync(storedValues, documents, scenario, account, this.OutputValues); } // Get the HttpRequest, either from MethodName or by parsing HttpRequest HttpRequest request; try { request = this.GetHttpRequest(documents); } catch (Exception ex) { errors.Add(new ValidationError(ValidationErrorCode.InvalidRequestFormat, null, "An error occured creating the http request: {0}", ex.Message)); return new ValidationResult<bool>(false, errors); } MethodDefinition.AddTestHeaderToRequest(scenario, request); MethodDefinition.AddAdditionalHeadersToRequest(account, request); // If this is a canned request, we need to merge the parameters / placeholders here var placeholderValues = this.RequestParameters.ToPlaceholderValuesArray(storedValues); // Update the request with the parameters in request-parameters try { request.RewriteRequestWithParameters(placeholderValues); } catch (Exception ex) { errors.Add(new ValidationError(ValidationErrorCode.ParameterParserError, SourceName, "Error rewriting the request with parameters from the scenario: {0}", ex.Message)); return new ValidationResult<bool>(false, errors); } MethodDefinition.AddAccessTokenToRequest(account.CreateCredentials(), request); errors.Add(new ValidationMessage(null, "Test-setup request:\n{0}", request.FullHttpText())); try { var response = await request.GetResponseAsync(account.BaseUrl); if (response.RetryCount > 0) { errors.Add(new ValidationWarning(ValidationErrorCode.RequestWasRetried, null, "HTTP request was retried {0} times.", response.RetryCount)); } errors.Add(new ValidationMessage(null, "HTTP Response:\n{0}\n\n", response.FullText())); // Check to see if this request is "successful" or not if ( (this.AllowedStatusCodes == null && response.WasSuccessful) || (this.AllowedStatusCodes != null && this.AllowedStatusCodes.Contains(response.StatusCode))) { string expectedContentType = (null != this.OutputValues) ? ExpectedResponseContentType(this.OutputValues.Values) : null; // Check for content type mismatch if (string.IsNullOrEmpty(response.ContentType) && expectedContentType != null) { return new ValidationResult<bool>(false, new ValidationError(ValidationErrorCode.UnsupportedContentType, SourceName, "No Content-Type found for a non-204 response")); } // Load requested values into stored values if (null != this.OutputValues) { foreach (var outputKey in this.OutputValues.Keys) { var source = this.OutputValues[outputKey]; storedValues[outputKey] = response.ValueForKeyedIdentifier(source); } } return new ValidationResult<bool>(!errors.Any(x => x.IsError), errors); } else { if ((this.AllowedStatusCodes != null && !this.AllowedStatusCodes.Contains(response.StatusCode)) || !response.WasSuccessful) { string expectedCodes = "200-299"; if (this.AllowedStatusCodes != null) expectedCodes = this.AllowedStatusCodes.ComponentsJoinedByString(","); errors.Add(new ValidationError(ValidationErrorCode.HttpStatusCodeDifferent, SourceName, "Http response status code {0} didn't match expected values: {1}", response.StatusCode, expectedCodes)); } else { errors.Add(new ValidationError(ValidationErrorCode.HttpStatusCodeDifferent, SourceName, "Http response content type was invalid: {0}", response.ContentType)); } return new ValidationResult<bool>(false, errors); } } catch (Exception ex) { errors.Add(new ValidationError(ValidationErrorCode.Unknown, SourceName, "Exception while making request: {0}", ex.Message)); return new ValidationResult<bool>(false, errors); } }
/// <summary> /// Validates that a particular HttpResponse matches the method definition and optionally the expected response. /// </summary> /// <param name="method">Method definition that was used to generate a request.</param> /// <param name="actualResponse">Actual response from the service (this is what we validate).</param> /// <param name="expectedResponse">Prototype response (expected) that shows what a valid response should look like.</param> /// <param name="scenario">A test scenario used to generate the response, which may include additional parameters to verify.</param> /// <param name="errors">A collection of errors, warnings, and verbose messages generated by this process.</param> public void ValidateResponse(HttpResponse actualResponse, HttpResponse expectedResponse, ScenarioDefinition scenario, out ValidationError[] errors, ValidationOptions options = null) { if (null == actualResponse) throw new ArgumentNullException("actualResponse"); List<ValidationError> detectedErrors = new List<ValidationError>(); // Verify the request is valid (headers, etc) this.VerifyHttpRequest(detectedErrors); // Verify that the expected response headers match the actual response headers ValidationError[] httpErrors; if (null != expectedResponse && !expectedResponse.ValidateResponseHeaders(actualResponse, out httpErrors, (null != scenario) ? scenario.AllowedStatusCodes : null)) { detectedErrors.AddRange(httpErrors); } // Verify the actual response body is correct according to the schema defined for the response ValidationError[] bodyErrors; this.VerifyResponseBody(actualResponse, expectedResponse, out bodyErrors, options); detectedErrors.AddRange(bodyErrors); // Verify any expectations in the scenario are met if (null != scenario) { scenario.ValidateExpectations(actualResponse, detectedErrors); } errors = detectedErrors.ToArray(); }
/// <summary> /// Add information about the test that generated this call to the request headers. /// </summary> /// <param name="scenario"></param> /// <param name="request"></param> internal static void AddTestHeaderToRequest(ScenarioDefinition scenario, HttpRequest request) { var headerValue = string.Format( "method-name: {0}; scenario-name: {1}", scenario.MethodName, scenario.Description); request.Headers.Add("ApiDocsTestInfo", headerValue); }
/// <summary> /// Take a scenario definition and convert the prototype request into a fully formed request. This includes appending /// the base URL to the request URL, executing any test-setup requests, and replacing the placeholders in the prototype /// request with proper values. /// </summary> /// <param name="scenario"></param> /// <param name="documents"></param> /// <param name="account"></param> /// <returns></returns> public async Task<ValidationResult<HttpRequest>> GenerateMethodRequestAsync(ScenarioDefinition scenario, DocSet documents, IServiceAccount account) { var parser = new HttpParser(); var request = parser.ParseHttpRequest(this.Request); AddAccessTokenToRequest(account.CreateCredentials(), request); AddTestHeaderToRequest(scenario, request); AddAdditionalHeadersToRequest(account, request); List<ValidationError> errors = new List<ValidationError>(); if (null != scenario) { var storedValuesForScenario = new Dictionary<string, string>(); if (null != scenario.TestSetupRequests) { foreach (var setupRequest in scenario.TestSetupRequests) { var result = await setupRequest.MakeSetupRequestAsync(storedValuesForScenario, documents, scenario, account); errors.AddRange(result.Messages); if (result.IsWarningOrError) { // If we can an error or warning back from a setup method, we fail the whole request. return new ValidationResult<HttpRequest>(null, errors); } } } try { var placeholderValues = scenario.RequestParameters.ToPlaceholderValuesArray(storedValuesForScenario); request.RewriteRequestWithParameters(placeholderValues); } catch (Exception ex) { // Error when applying parameters to the request errors.Add( new ValidationError( ValidationErrorCode.RewriteRequestFailure, "GenerateMethodRequestAsync", ex.Message)); return new ValidationResult<HttpRequest>(null, errors); } if (scenario.StatusCodesToRetry != null) { request.RetryOnStatusCode = (from status in scenario.StatusCodesToRetry select (System.Net.HttpStatusCode)status).ToList(); } } if (string.IsNullOrEmpty(request.Accept)) { if (!string.IsNullOrEmpty(ValidationConfig.ODataMetadataLevel)) { request.Accept = MimeTypeJson + "; " + ValidationConfig.ODataMetadataLevel; } else { request.Accept = MimeTypeJson; } } return new ValidationResult<HttpRequest>(request, errors); }
/// <summary> /// Make a call to the HttpRequest and populate the Value property of /// every PlaceholderValue in Values based on the result. /// </summary> /// <param name="baseUrl"></param> /// <param name="credentials"></param> /// <param name="storedValues"></param> /// <param name="documents"></param> /// <returns></returns> public async Task <ValidationResult <bool> > MakeSetupRequestAsync(string baseUrl, AuthenicationCredentials credentials, Dictionary <string, string> storedValues, DocSet documents, ScenarioDefinition scenario) { var errors = new List <ValidationError>(); if (!string.IsNullOrEmpty(this.CannedRequestName)) { // We need to make a canned request setup request instead. Look up the canned request and then execute it, returning the results here. var cannedRequest = (from cr in documents.CannedRequests where cr.Name == this.CannedRequestName select cr) .FirstOrDefault(); if (null == cannedRequest) { errors.Add(new ValidationError(ValidationErrorCode.InvalidRequestFormat, null, "Couldn't locate the canned-request named: {0}", this.CannedRequestName)); return(new ValidationResult <bool>(false, errors)); } return(await cannedRequest.MakeSetupRequestAsync(baseUrl, credentials, storedValues, documents, scenario)); } // Get the HttpRequest, either from MethodName or by parsing HttpRequest HttpRequest request; try { request = this.GetHttpRequest(documents); } catch (Exception ex) { errors.Add(new ValidationError(ValidationErrorCode.InvalidRequestFormat, null, "An error occured creating the http request: {0}", ex.Message)); return(new ValidationResult <bool>(false, errors)); } MethodDefinition.AddTestHeaderToRequest(scenario, request); // If this is a canned request, we need to merge the parameters / placeholders here var placeholderValues = this.RequestParameters.ToPlaceholderValuesArray(storedValues); // Update the request with the parameters in request-parameters try { request.RewriteRequestWithParameters(placeholderValues); } catch (Exception ex) { errors.Add(new ValidationError(ValidationErrorCode.ParameterParserError, SourceName, "Error rewriting the request with parameters from the scenario: {0}", ex.Message)); return(new ValidationResult <bool>(false, errors)); } MethodDefinition.AddAccessTokenToRequest(credentials, request); errors.Add(new ValidationMessage(null, "Test-setup request:\n{0}", request.FullHttpText())); try { var response = await request.GetResponseAsync(baseUrl); if (response.RetryCount > 0) { errors.Add(new ValidationWarning(ValidationErrorCode.RequestWasRetried, null, "HTTP request was retried {0} times.", response.RetryCount)); } errors.Add(new ValidationMessage(null, "HTTP Response:\n{0}\n\n", response.FullText())); // Check to see if this request is "successful" or not if ((this.AllowedStatusCodes == null && response.WasSuccessful) || (this.AllowedStatusCodes != null && this.AllowedStatusCodes.Contains(response.StatusCode))) { string expectedContentType = (null != this.OutputValues) ? ExpectedResponseContentType(this.OutputValues.Values) : null; // Check for content type mismatch if (string.IsNullOrEmpty(response.ContentType) && expectedContentType != null) { return(new ValidationResult <bool>(false, new ValidationError(ValidationErrorCode.UnsupportedContentType, SourceName, "No Content-Type found for a non-204 response"))); } // Load requested values into stored values if (null != this.OutputValues) { foreach (var outputKey in this.OutputValues.Keys) { var source = this.OutputValues[outputKey]; storedValues[outputKey] = response.ValueForKeyedIdentifier(source); } } return(new ValidationResult <bool>(!errors.Any(x => x.IsError), errors)); } else { if ((this.AllowedStatusCodes != null && !this.AllowedStatusCodes.Contains(response.StatusCode)) || !response.WasSuccessful) { string expectedCodes = "200-299"; if (this.AllowedStatusCodes != null) { expectedCodes = this.AllowedStatusCodes.ComponentsJoinedByString(","); } errors.Add(new ValidationError(ValidationErrorCode.HttpStatusCodeDifferent, SourceName, "Http response status code {0} didn't match expected values: {1}", response.StatusCode, expectedCodes)); } else { errors.Add(new ValidationError(ValidationErrorCode.HttpStatusCodeDifferent, SourceName, "Http response content type was invalid: {0}", response.ContentType)); } return(new ValidationResult <bool>(false, errors)); } } catch (Exception ex) { errors.Add(new ValidationError(ValidationErrorCode.Unknown, SourceName, "Exception while making request: {0}", ex.Message)); return(new ValidationResult <bool>(false, errors)); } }