/// <summary> /// Execute the provided methods on the given account. /// </summary> /// <param name="options"></param> /// <param name="account"></param> /// <param name="methods"></param> /// <param name="docset"></param> /// <returns>True if the methods all passed, false if there were failures.</returns> private static async Task <bool> CheckMethodsForAccountAsync(CheckServiceOptions options, IServiceAccount account, MethodDefinition[] methods, DocSet docset) { //CheckResults results = new CheckResults(); ConfigureAdditionalHeadersForAccount(options, account); string testNamePrefix = account.Name.ToLower() + ": "; FancyConsole.WriteLine(FancyConsole.ConsoleHeaderColor, "Testing with account: {0}", account.Name); FancyConsole.WriteLine(FancyConsole.ConsoleCodeColor, "Preparing authentication for requests...", account.Name); try { await account.PrepareForRequestAsync(); } catch (Exception ex) { RecordError(ex.Message); return(false); } AuthenicationCredentials credentials = account.CreateCredentials(); int concurrentTasks = options.ParallelTests ? ParallelTaskCount : 1; CheckResults docSetResults = new CheckResults(); await ForEachAsync(methods, concurrentTasks, async method => { FancyConsole.WriteLine( FancyConsole.ConsoleCodeColor, "Running validation for method: {0}", method.Identifier); ScenarioDefinition[] scenarios = docset.TestScenarios.ScenariosForMethod(method); ValidationResults results = await method.ValidateServiceResponseAsync(scenarios, account, credentials); PrintResultsToConsole(method, account, results, options); await TestReport.LogMethodTestResults(method, account, results); docSetResults.RecordResults(results, options); if (concurrentTasks == 1) { AddPause(options); } }); if (options.IgnoreWarnings || options.SilenceWarnings) { // Remove the warning flag from the outcomes docSetResults.ConvertWarningsToSuccess(); } docSetResults.PrintToConsole(); bool hadWarnings = docSetResults.WarningCount > 0; bool hadErrors = docSetResults.FailureCount > 0; return(!(hadErrors | hadWarnings)); }
/// <summary> /// Make a call to the HttpRequest and populate the Value property of /// every PlaceholderValue in Values based on the result. /// </summary> /// <param name="storedValues"></param> /// <param name="documents"></param> /// <param name="scenario"></param> /// <param name="account"></param> /// <param name="parentOutputValues"></param> /// <returns></returns> public async Task<ValidationResult<bool>> MakeSetupRequestAsync(Dictionary<string, string> storedValues, DocSet documents, ScenarioDefinition scenario, IServiceAccount account, Dictionary<string, string> parentOutputValues = null) { // Copy the output values from parentOutputValues into our own if (null != parentOutputValues) { foreach (var key in parentOutputValues.Keys) { this.OutputValues.Add(key, parentOutputValues[key]); } } var errors = new List<ValidationError>(); if (!string.IsNullOrEmpty(this.CannedRequestName)) { // We need to make a canned request setup request instead. Look up the canned request and then execute it, returning the results here. var cannedRequest = (from cr in documents.CannedRequests where cr.Name == this.CannedRequestName select cr) .FirstOrDefault(); if (null == cannedRequest) { errors.Add(new ValidationError(ValidationErrorCode.InvalidRequestFormat, null, "Couldn't locate the canned-request named: {0}", this.CannedRequestName)); return new ValidationResult<bool>(false, errors); } // Need to make a copy of the canned request here so that our state doesn't continue to pile up var cannedRequestInstance = cannedRequest.CopyInstance(); return await cannedRequestInstance.MakeSetupRequestAsync(storedValues, documents, scenario, account, this.OutputValues); } // Get the HttpRequest, either from MethodName or by parsing HttpRequest HttpRequest request; try { request = this.GetHttpRequest(documents); } catch (Exception ex) { errors.Add(new ValidationError(ValidationErrorCode.InvalidRequestFormat, null, "An error occured creating the http request: {0}", ex.Message)); return new ValidationResult<bool>(false, errors); } MethodDefinition.AddTestHeaderToRequest(scenario, request); MethodDefinition.AddAdditionalHeadersToRequest(account, request); // If this is a canned request, we need to merge the parameters / placeholders here var placeholderValues = this.RequestParameters.ToPlaceholderValuesArray(storedValues); // Update the request with the parameters in request-parameters try { request.RewriteRequestWithParameters(placeholderValues); } catch (Exception ex) { errors.Add(new ValidationError(ValidationErrorCode.ParameterParserError, SourceName, "Error rewriting the request with parameters from the scenario: {0}", ex.Message)); return new ValidationResult<bool>(false, errors); } MethodDefinition.AddAccessTokenToRequest(account.CreateCredentials(), request); errors.Add(new ValidationMessage(null, "Test-setup request:\n{0}", request.FullHttpText())); try { var response = await request.GetResponseAsync(account.BaseUrl); if (response.RetryCount > 0) { errors.Add(new ValidationWarning(ValidationErrorCode.RequestWasRetried, null, "HTTP request was retried {0} times.", response.RetryCount)); } errors.Add(new ValidationMessage(null, "HTTP Response:\n{0}\n\n", response.FullText())); // Check to see if this request is "successful" or not if ( (this.AllowedStatusCodes == null && response.WasSuccessful) || (this.AllowedStatusCodes != null && this.AllowedStatusCodes.Contains(response.StatusCode))) { string expectedContentType = (null != this.OutputValues) ? ExpectedResponseContentType(this.OutputValues.Values) : null; // Check for content type mismatch if (string.IsNullOrEmpty(response.ContentType) && expectedContentType != null) { return new ValidationResult<bool>(false, new ValidationError(ValidationErrorCode.UnsupportedContentType, SourceName, "No Content-Type found for a non-204 response")); } // Load requested values into stored values if (null != this.OutputValues) { foreach (var outputKey in this.OutputValues.Keys) { var source = this.OutputValues[outputKey]; storedValues[outputKey] = response.ValueForKeyedIdentifier(source); } } return new ValidationResult<bool>(!errors.Any(x => x.IsError), errors); } else { if ((this.AllowedStatusCodes != null && !this.AllowedStatusCodes.Contains(response.StatusCode)) || !response.WasSuccessful) { string expectedCodes = "200-299"; if (this.AllowedStatusCodes != null) expectedCodes = this.AllowedStatusCodes.ComponentsJoinedByString(","); errors.Add(new ValidationError(ValidationErrorCode.HttpStatusCodeDifferent, SourceName, "Http response status code {0} didn't match expected values: {1}", response.StatusCode, expectedCodes)); } else { errors.Add(new ValidationError(ValidationErrorCode.HttpStatusCodeDifferent, SourceName, "Http response content type was invalid: {0}", response.ContentType)); } return new ValidationResult<bool>(false, errors); } } catch (Exception ex) { errors.Add(new ValidationError(ValidationErrorCode.Unknown, SourceName, "Exception while making request: {0}", ex.Message)); return new ValidationResult<bool>(false, errors); } }
/// <summary> /// Take a scenario definition and convert the prototype request into a fully formed request. This includes appending /// the base URL to the request URL, executing any test-setup requests, and replacing the placeholders in the prototype /// request with proper values. /// </summary> /// <param name="scenario"></param> /// <param name="documents"></param> /// <param name="account"></param> /// <returns></returns> public async Task<ValidationResult<HttpRequest>> GenerateMethodRequestAsync(ScenarioDefinition scenario, DocSet documents, IServiceAccount account) { var parser = new HttpParser(); var request = parser.ParseHttpRequest(this.Request); AddAccessTokenToRequest(account.CreateCredentials(), request); AddTestHeaderToRequest(scenario, request); AddAdditionalHeadersToRequest(account, request); List<ValidationError> errors = new List<ValidationError>(); if (null != scenario) { var storedValuesForScenario = new Dictionary<string, string>(); if (null != scenario.TestSetupRequests) { foreach (var setupRequest in scenario.TestSetupRequests) { var result = await setupRequest.MakeSetupRequestAsync(storedValuesForScenario, documents, scenario, account); errors.AddRange(result.Messages); if (result.IsWarningOrError) { // If we can an error or warning back from a setup method, we fail the whole request. return new ValidationResult<HttpRequest>(null, errors); } } } try { var placeholderValues = scenario.RequestParameters.ToPlaceholderValuesArray(storedValuesForScenario); request.RewriteRequestWithParameters(placeholderValues); } catch (Exception ex) { // Error when applying parameters to the request errors.Add( new ValidationError( ValidationErrorCode.RewriteRequestFailure, "GenerateMethodRequestAsync", ex.Message)); return new ValidationResult<HttpRequest>(null, errors); } if (scenario.StatusCodesToRetry != null) { request.RetryOnStatusCode = (from status in scenario.StatusCodesToRetry select (System.Net.HttpStatusCode)status).ToList(); } } if (string.IsNullOrEmpty(request.Accept)) { if (!string.IsNullOrEmpty(ValidationConfig.ODataMetadataLevel)) { request.Accept = MimeTypeJson + "; " + ValidationConfig.ODataMetadataLevel; } else { request.Accept = MimeTypeJson; } } return new ValidationResult<HttpRequest>(request, errors); }
/// <summary> /// Take a scenario definition and convert the prototype request into a fully formed request. This includes appending /// the base URL to the request URL, executing any test-setup requests, and replacing the placeholders in the prototype /// request with proper values. /// </summary> /// <param name="scenario"></param> /// <param name="documents"></param> /// <param name="account"></param> /// <returns></returns> public async Task <ValidationResult <HttpRequest> > GenerateMethodRequestAsync(ScenarioDefinition scenario, DocSet documents, IServiceAccount account) { var parser = new HttpParser(); var request = parser.ParseHttpRequest(this.Request); AddAccessTokenToRequest(account.CreateCredentials(), request); AddTestHeaderToRequest(scenario, request); AddAdditionalHeadersToRequest(account, request); List <ValidationError> errors = new List <ValidationError>(); if (null != scenario) { var storedValuesForScenario = new Dictionary <string, string>(); if (null != scenario.TestSetupRequests) { foreach (var setupRequest in scenario.TestSetupRequests) { try { var result = await setupRequest.MakeSetupRequestAsync(storedValuesForScenario, documents, scenario, account); errors.AddRange(result.Messages); if (result.IsWarningOrError) { // If we can an error or warning back from a setup method, we fail the whole request. return(new ValidationResult <HttpRequest>(null, errors)); } } catch (Exception ex) { return(new ValidationResult <HttpRequest>(null, new ValidationError(ValidationErrorCode.ConsolidatedError, null, "An exception occured while processing setup-requests: {0}", ex.Message))); } } } try { var placeholderValues = scenario.RequestParameters.ToPlaceholderValuesArray(storedValuesForScenario); request.RewriteRequestWithParameters(placeholderValues); } catch (Exception ex) { // Error when applying parameters to the request errors.Add( new ValidationError( ValidationErrorCode.RewriteRequestFailure, "GenerateMethodRequestAsync", ex.Message)); return(new ValidationResult <HttpRequest>(null, errors)); } if (scenario.StatusCodesToRetry != null) { request.RetryOnStatusCode = (from status in scenario.StatusCodesToRetry select(System.Net.HttpStatusCode) status).ToList(); } } if (string.IsNullOrEmpty(request.Accept)) { if (!string.IsNullOrEmpty(ValidationConfig.ODataMetadataLevel)) { request.Accept = MimeTypeJson + "; " + ValidationConfig.ODataMetadataLevel; } else { request.Accept = MimeTypeJson; } } this.ModifyRequestForAccount(request, account); return(new ValidationResult <HttpRequest>(request, errors)); }
/// <summary> /// Make a call to the HttpRequest and populate the Value property of /// every PlaceholderValue in Values based on the result. /// </summary> /// <param name="storedValues"></param> /// <param name="documents"></param> /// <param name="scenario"></param> /// <param name="account"></param> /// <param name="parentOutputValues"></param> /// <returns></returns> public async Task <ValidationResult <bool> > MakeSetupRequestAsync( Dictionary <string, string> storedValues, DocSet documents, ScenarioDefinition scenario, IServiceAccount account, Dictionary <string, string> parentOutputValues = null) { // Copy the output values from parentOutputValues into our own if (null != parentOutputValues) { foreach (var key in parentOutputValues.Keys) { this.OutputValues.Add(key, parentOutputValues[key]); } } var errors = new List <ValidationError>(); if (!string.IsNullOrEmpty(this.CannedRequestName)) { // We need to make a canned request setup request instead. Look up the canned request and then execute it, returning the results here. var cannedRequest = (from cr in documents.CannedRequests where cr.Name == this.CannedRequestName select cr) .FirstOrDefault(); if (null == cannedRequest) { errors.Add(new ValidationError(ValidationErrorCode.InvalidRequestFormat, null, "Couldn't locate the canned-request named: {0}", this.CannedRequestName)); return(new ValidationResult <bool>(false, errors)); } // Need to make a copy of the canned request here so that our state doesn't continue to pile up var cannedRequestInstance = cannedRequest.CopyInstance(); return(await cannedRequestInstance.MakeSetupRequestAsync(storedValues, documents, scenario, account, this.OutputValues)); } // Get the HttpRequest, either from MethodName or by parsing HttpRequest HttpRequest request; var issues = new IssueLogger(); try { request = this.GetHttpRequest(documents, account, issues); } catch (Exception ex) { errors.Add(new ValidationError(ValidationErrorCode.InvalidRequestFormat, null, "An error occured creating the http request: {0}", ex.Message)); return(new ValidationResult <bool>(false, errors)); } MethodDefinition.AddTestHeaderToRequest(scenario, request); MethodDefinition.AddAdditionalHeadersToRequest(account, request); // If this is a canned request, we need to merge the parameters / placeholders here var placeholderValues = this.RequestParameters.ToPlaceholderValuesArray(storedValues); // Update the request with the parameters in request-parameters try { request.RewriteRequestWithParameters(placeholderValues); } catch (Exception ex) { errors.Add(new ValidationError(ValidationErrorCode.ParameterParserError, SourceName, "Error rewriting the request with parameters from the scenario: {0}", ex.Message)); return(new ValidationResult <bool>(false, errors)); } MethodDefinition.AddAccessTokenToRequest(account.CreateCredentials(), request); errors.Add(new ValidationMessage(null, "Test-setup request:\n{0}", request.FullHttpText())); try { var response = await request.GetResponseAsync(account, issues); errors.AddRange(issues.Issues); if (response.RetryCount > 0) { errors.Add(new ValidationWarning(ValidationErrorCode.RequestWasRetried, null, "HTTP request was retried {0} times.", response.RetryCount)); } errors.Add(new ValidationMessage(null, "HTTP Response:\n{0}\n\n", response.FullText())); // Check to see if this request is "successful" or not if ((this.AllowedStatusCodes == null && response.WasSuccessful) || (this.AllowedStatusCodes != null && this.AllowedStatusCodes.Contains(response.StatusCode))) { string expectedContentType = (null != this.OutputValues) ? ExpectedResponseContentType(this.OutputValues.Values) : null; // Check for content type mismatch if (string.IsNullOrEmpty(response.ContentType) && expectedContentType != null) { return(new ValidationResult <bool>(false, new ValidationError(ValidationErrorCode.UnsupportedContentType, SourceName, "No Content-Type found for a non-204 response"))); } // Load requested values into stored values if (null != this.OutputValues) { foreach (var outputKey in this.OutputValues.Keys) { var source = this.OutputValues[outputKey]; storedValues[outputKey] = response.ValueForKeyedIdentifier(source); } } return(new ValidationResult <bool>(!errors.Any(x => x.IsError), errors)); } else { if ((this.AllowedStatusCodes != null && !this.AllowedStatusCodes.Contains(response.StatusCode)) || !response.WasSuccessful) { string expectedCodes = "200-299"; if (this.AllowedStatusCodes != null) { expectedCodes = this.AllowedStatusCodes.ComponentsJoinedByString(","); } errors.Add(new ValidationError(ValidationErrorCode.HttpStatusCodeDifferent, SourceName, "Http response status code {0} didn't match expected values: {1}", response.StatusCode, expectedCodes)); } else { errors.Add(new ValidationError(ValidationErrorCode.HttpStatusCodeDifferent, SourceName, "Http response content type was invalid: {0}", response.ContentType)); } return(new ValidationResult <bool>(false, errors)); } } catch (Exception ex) { errors.Add(new ValidationError(ValidationErrorCode.Unknown, SourceName, "Exception while making request: {0}", ex.Message)); return(new ValidationResult <bool>(false, errors)); } }