/// <summary> /// Performs validation of a method (request/response) with a given service /// target and zero or more test scenarios /// </summary> /// <param name="method"></param> /// <param name="account"></param> /// <param name="credentials"></param> /// <returns></returns> public static async Task <ValidationResults> ValidateServiceResponseAsync( this MethodDefinition method, ScenarioDefinition[] scenarios, IServiceAccount account, AuthenicationCredentials credentials) { if (null == method) { throw new ArgumentNullException("method"); } if (null == account) { throw new ArgumentNullException("account"); } if (null == credentials) { throw new ArgumentNullException("credentials"); } ValidationResults results = new ValidationResults(); if (scenarios.Length == 0) { // If no descenarios are defined for this method, add a new default scenario scenarios = new ScenarioDefinition[] { new ScenarioDefinition { Description = "default-scenario", Enabled = true, MethodName = method.Identifier } }; results.AddResult("init", new ValidationMessage(null, "No scenarios were defined for method {0}. Will execute request as written.", method.Identifier), ValidationOutcome.None); } if (scenarios.Any() && !scenarios.Any(x => x.Enabled)) { results.AddResult("init", new ValidationWarning(ValidationErrorCode.AllScenariosDisabled, null, "All scenarios for method {0} were disabled.", method.Identifier), ValidationOutcome.Skipped); return(results); } foreach (var scenario in scenarios.Where(x => x.Enabled)) { try { await ValidateMethodWithScenarioAsync(method, scenario, account, credentials, results); } catch (Exception ex) { results.AddResult( "validation", new ValidationError( ValidationErrorCode.ExceptionWhileValidatingMethod, method.SourceFile.DisplayName, ex.Message)); } } return(results); }
/// <summary> /// Performs validation of a method (request/response) with a given service /// target and zero or more test scenarios /// </summary> /// <param name="method"></param> /// <param name="account"></param> /// <param name="credentials"></param> /// <returns></returns> public static async Task<ValidationResults> ValidateServiceResponseAsync( this MethodDefinition method, ScenarioDefinition[] scenarios, IServiceAccount account, AuthenicationCredentials credentials) { if (null == method) throw new ArgumentNullException("method"); if (null == account) throw new ArgumentNullException("account"); if (null == credentials) throw new ArgumentNullException("credentials"); ValidationResults results = new ValidationResults(); if (scenarios.Length == 0) { // If no descenarios are defined for this method, add a new default scenario scenarios = new ScenarioDefinition[] { new ScenarioDefinition { Description = "default-scenario", Enabled = true, MethodName = method.Identifier} }; results.AddResult("init", new ValidationMessage(null, "No scenarios were defined for method {0}. Will execute request as written.", method.Identifier), ValidationOutcome.None); } if (scenarios.Any() && !scenarios.Any(x => x.Enabled)) { results.AddResult("init", new ValidationWarning(ValidationErrorCode.AllScenariosDisabled, null, "All scenarios for method {0} were disabled.", method.Identifier), ValidationOutcome.Skipped); return results; } foreach (var scenario in scenarios.Where(x => x.Enabled)) { try { await ValidateMethodWithScenarioAsync(method, scenario, account, credentials, results); } catch (Exception ex) { results.AddResult( "validation", new ValidationError( ValidationErrorCode.ExceptionWhileValidatingMethod, method.SourceFile.DisplayName, ex.Message)); } } return results; }
private static async Task ValidateMethodWithScenarioAsync( MethodDefinition method, ScenarioDefinition scenario, IServiceAccount account, AuthenicationCredentials credentials, ValidationResults results) { if (null == method) throw new ArgumentNullException("method"); if (null == scenario) throw new ArgumentNullException("scenario"); if (null == account) throw new ArgumentNullException("account"); if (null == credentials) throw new ArgumentNullException("credentials"); if (null == results) throw new ArgumentNullException("results"); var actionName = scenario.Description; // Generate the tested request by "previewing" the request and executing // all test-setup procedures long startTicks = DateTimeOffset.UtcNow.Ticks; var requestPreviewResult = await method.GenerateMethodRequestAsync(scenario, account.BaseUrl, credentials, method.SourceFile.Parent); TimeSpan generateMethodDuration = new TimeSpan(DateTimeOffset.UtcNow.Ticks - startTicks); // Check to see if an error occured building the request, and abort if so. var generatorResults = results[actionName + " [test-setup requests]"]; generatorResults.AddResults(requestPreviewResult.Messages, requestPreviewResult.IsWarningOrError ? ValidationOutcome.Error : ValidationOutcome.Passed); generatorResults.Duration = generateMethodDuration; if (requestPreviewResult.IsWarningOrError) { return; } // We've done all the test-setup work, now we have the real request to make to the service HttpRequest requestPreview = requestPreviewResult.Value; results.AddResult( actionName, new ValidationMessage(null, "Generated Method HTTP Request:\r\n{0}", requestPreview.FullHttpText())); HttpParser parser = new HttpParser(); HttpResponse expectedResponse = null; if (!string.IsNullOrEmpty(method.ExpectedResponse)) { expectedResponse = parser.ParseHttpResponse(method.ExpectedResponse); } // Execute the actual tested method (the result of the method preview call, which made the test-setup requests) startTicks = DateTimeOffset.UtcNow.Ticks; var actualResponse = await requestPreview.GetResponseAsync(account.BaseUrl); TimeSpan actualMethodDuration = new TimeSpan(DateTimeOffset.UtcNow.Ticks - startTicks); var requestResults = results[actionName]; if (actualResponse.RetryCount > 0) { requestResults.AddResults( new ValidationError[] { new ValidationWarning(ValidationErrorCode.RequestWasRetried, null, "HTTP request was retried {0} times.", actualResponse.RetryCount) }); } requestResults.AddResults( new ValidationError[] { new ValidationMessage(null, "HTTP Response:\r\n{0}", actualResponse.FullText(false)) }); requestResults.Duration = actualMethodDuration; // Perform validation on the method's actual response ValidationError[] errors; method.ValidateResponse(actualResponse, expectedResponse, scenario, out errors); requestResults.AddResults(errors); // TODO: If the method is defined as a long running operation, we need to go poll the status // URL to make sure that the operation finished and the response type is valid. if (errors.WereErrors()) results.SetOutcome(actionName, ValidationOutcome.Error); else if (errors.WereWarnings()) results.SetOutcome(actionName, ValidationOutcome.Warning); else results.SetOutcome(actionName, ValidationOutcome.Passed); }
private static async Task ValidateMethodWithScenarioAsync( MethodDefinition method, ScenarioDefinition scenario, IServiceAccount account, AuthenicationCredentials credentials, ValidationResults results, ValidationOptions options = null) { if (null == method) { throw new ArgumentNullException("method"); } if (null == scenario) { throw new ArgumentNullException("scenario"); } if (null == account) { throw new ArgumentNullException("account"); } if (null == credentials) { throw new ArgumentNullException("credentials"); } if (null == results) { throw new ArgumentNullException("results"); } var actionName = scenario.Description; // Check to see if the account + scenario scopes are aligned string[] requiredScopes = method.RequiredScopes.Union(scenario.RequiredScopes).ToArray(); if (!account.Scopes.ProvidesScopes(requiredScopes, options.IgnoreRequiredScopes)) { var missingScopes = from scope in requiredScopes where !account.Scopes.Contains(scope) select scope; results.AddResult(actionName, new ValidationWarning(ValidationErrorCode.RequiredScopesMissing, null, "Scenario was not run. Scopes required were not available: {0}", missingScopes.ComponentsJoinedByString(","))); return; } // Generate the tested request by "previewing" the request and executing // all test-setup procedures long startTicks = DateTimeOffset.UtcNow.Ticks; var requestPreviewResult = await method.GenerateMethodRequestAsync(scenario, account.BaseUrl, credentials, method.SourceFile.Parent); TimeSpan generateMethodDuration = new TimeSpan(DateTimeOffset.UtcNow.Ticks - startTicks); // Check to see if an error occured building the request, and abort if so. var generatorResults = results[actionName /*+ " [test-setup requests]"*/]; generatorResults.AddResults(requestPreviewResult.Messages, requestPreviewResult.IsWarningOrError ? ValidationOutcome.Error : ValidationOutcome.Passed); generatorResults.Duration = generateMethodDuration; if (requestPreviewResult.IsWarningOrError) { return; } // We've done all the test-setup work, now we have the real request to make to the service HttpRequest requestPreview = requestPreviewResult.Value; results.AddResult( actionName, new ValidationMessage(null, "Generated Method HTTP Request:\r\n{0}", requestPreview.FullHttpText())); HttpParser parser = new HttpParser(); HttpResponse expectedResponse = null; if (!string.IsNullOrEmpty(method.ExpectedResponse)) { expectedResponse = parser.ParseHttpResponse(method.ExpectedResponse); } // Execute the actual tested method (the result of the method preview call, which made the test-setup requests) startTicks = DateTimeOffset.UtcNow.Ticks; var actualResponse = await requestPreview.GetResponseAsync(account.BaseUrl); TimeSpan actualMethodDuration = new TimeSpan(DateTimeOffset.UtcNow.Ticks - startTicks); var requestResults = results[actionName]; if (actualResponse.RetryCount > 0) { requestResults.AddResults( new ValidationError[] { new ValidationWarning(ValidationErrorCode.RequestWasRetried, null, "HTTP request was retried {0} times.", actualResponse.RetryCount) }); } requestResults.AddResults( new ValidationError[] { new ValidationMessage(null, "HTTP Response:\r\n{0}", actualResponse.FullText(false)) }); requestResults.Duration = actualMethodDuration; // Perform validation on the method's actual response ValidationError[] errors; method.ValidateResponse(actualResponse, expectedResponse, scenario, out errors, options); requestResults.AddResults(errors); // TODO: If the method is defined as a long running operation, we need to go poll the status // URL to make sure that the operation finished and the response type is valid. if (errors.WereErrors()) { results.SetOutcome(actionName, ValidationOutcome.Error); } else if (errors.WereWarnings()) { results.SetOutcome(actionName, ValidationOutcome.Warning); } else { results.SetOutcome(actionName, ValidationOutcome.Passed); } }
internal static void AddAccessTokenToRequest(AuthenicationCredentials credentials, HttpRequest request) { credentials.AuthenticateRequest(request); }
/// <summary> /// Take a scenario definition and convert the prototype request into a fully formed request. This includes appending /// the base URL to the request URL, executing any test-setup requests, and replacing the placeholders in the prototype /// request with proper values. /// </summary> /// <param name="scenario"></param> /// <param name="baseUrl"></param> /// <param name="credentials"></param> /// <param name="documents"></param> /// <returns></returns> public async Task <ValidationResult <HttpRequest> > GenerateMethodRequestAsync(ScenarioDefinition scenario, string baseUrl, AuthenicationCredentials credentials, DocSet documents) { var parser = new HttpParser(); var request = parser.ParseHttpRequest(this.Request); AddAccessTokenToRequest(credentials, request); AddTestHeaderToRequest(scenario, request); List <ValidationError> errors = new List <ValidationError>(); if (null != scenario) { var storedValuesForScenario = new Dictionary <string, string>(); if (null != scenario.TestSetupRequests) { foreach (var setupRequest in scenario.TestSetupRequests) { var result = await setupRequest.MakeSetupRequestAsync(baseUrl, credentials, storedValuesForScenario, documents, scenario); errors.AddRange(result.Messages); if (result.IsWarningOrError) { // If we can an error or warning back from a setup method, we fail the whole request. return(new ValidationResult <HttpRequest>(null, errors)); } } } try { var placeholderValues = scenario.RequestParameters.ToPlaceholderValuesArray(storedValuesForScenario); request.RewriteRequestWithParameters(placeholderValues); } catch (Exception ex) { // Error when applying parameters to the request errors.Add( new ValidationError( ValidationErrorCode.RewriteRequestFailure, "GenerateMethodRequestAsync", ex.Message)); return(new ValidationResult <HttpRequest>(null, errors)); } if (scenario.StatusCodesToRetry != null) { request.RetryOnStatusCode = (from status in scenario.StatusCodesToRetry select(System.Net.HttpStatusCode) status).ToList(); } } if (string.IsNullOrEmpty(request.Accept)) { if (!string.IsNullOrEmpty(ValidationConfig.ODataMetadataLevel)) { request.Accept = MimeTypeJson + "; " + ValidationConfig.ODataMetadataLevel; } else { request.Accept = MimeTypeJson; } } return(new ValidationResult <HttpRequest>(request, errors)); }
/// <summary> /// Take a scenario definition and convert the prototype request into a fully formed request. This includes appending /// the base URL to the request URL, executing any test-setup requests, and replacing the placeholders in the prototype /// request with proper values. /// </summary> /// <param name="scenario"></param> /// <param name="baseUrl"></param> /// <param name="credentials"></param> /// <param name="documents"></param> /// <returns></returns> public async Task<ValidationResult<HttpRequest>> GenerateMethodRequestAsync(ScenarioDefinition scenario, string baseUrl, AuthenicationCredentials credentials, DocSet documents) { var parser = new HttpParser(); var request = parser.ParseHttpRequest(this.Request); AddAccessTokenToRequest(credentials, request); AddTestHeaderToRequest(scenario, request); List<ValidationError> errors = new List<ValidationError>(); if (null != scenario) { var storedValuesForScenario = new Dictionary<string, string>(); if (null != scenario.TestSetupRequests) { foreach (var setupRequest in scenario.TestSetupRequests) { var result = await setupRequest.MakeSetupRequestAsync(baseUrl, credentials, storedValuesForScenario, documents, scenario); errors.AddRange(result.Messages); if (result.IsWarningOrError) { // If we can an error or warning back from a setup method, we fail the whole request. return new ValidationResult<HttpRequest>(null, errors); } } } try { var placeholderValues = scenario.RequestParameters.ToPlaceholderValuesArray(storedValuesForScenario); request.RewriteRequestWithParameters(placeholderValues); } catch (Exception ex) { // Error when applying parameters to the request errors.Add( new ValidationError( ValidationErrorCode.RewriteRequestFailure, "GenerateMethodRequestAsync", ex.Message)); return new ValidationResult<HttpRequest>(null, errors); } if (scenario.StatusCodesToRetry != null) { request.RetryOnStatusCode = (from status in scenario.StatusCodesToRetry select (System.Net.HttpStatusCode)status).ToList(); } } if (string.IsNullOrEmpty(request.Accept)) { if (!string.IsNullOrEmpty(ValidationConfig.ODataMetadataLevel)) { request.Accept = MimeTypeJson + "; " + ValidationConfig.ODataMetadataLevel; } else { request.Accept = MimeTypeJson; } } return new ValidationResult<HttpRequest>(request, errors); }