/// <summary>
/// Performs validation of a method (request/response) with a given service
/// target and zero or more test scenarios
/// </summary>
/// <param name="method"></param>
/// <param name="account"></param>
/// <param name="credentials"></param>
/// <returns></returns>
public static async Task <ValidationResults> ValidateServiceResponseAsync(
this MethodDefinition method,
ScenarioDefinition[] scenarios,
IServiceAccount account,
AuthenicationCredentials credentials)
{
if (null == method)
{
throw new ArgumentNullException("method");
}
if (null == account)
{
throw new ArgumentNullException("account");
}
if (null == credentials)
{
throw new ArgumentNullException("credentials");
}
ValidationResults results = new ValidationResults();
if (scenarios.Length == 0)
{
// If no descenarios are defined for this method, add a new default scenario
scenarios = new ScenarioDefinition[] { new ScenarioDefinition {
Description = "default-scenario", Enabled = true, MethodName = method.Identifier
} };
results.AddResult("init", new ValidationMessage(null, "No scenarios were defined for method {0}. Will execute request as written.", method.Identifier), ValidationOutcome.None);
}
if (scenarios.Any() && !scenarios.Any(x => x.Enabled))
{
results.AddResult("init",
new ValidationWarning(ValidationErrorCode.AllScenariosDisabled, null, "All scenarios for method {0} were disabled.", method.Identifier),
ValidationOutcome.Skipped);
return(results);
}
foreach (var scenario in scenarios.Where(x => x.Enabled))
{
try
{
await ValidateMethodWithScenarioAsync(method, scenario, account, credentials, results);
}
catch (Exception ex)
{
results.AddResult(
"validation",
new ValidationError(
ValidationErrorCode.ExceptionWhileValidatingMethod,
method.SourceFile.DisplayName,
ex.Message));
}
}
return(results);
}
/// <summary>
/// Performs validation of a method (request/response) with a given service
/// target and zero or more test scenarios
/// </summary>
/// <param name="method"></param>
/// <param name="account"></param>
/// <param name="credentials"></param>
/// <returns></returns>
public static async Task<ValidationResults> ValidateServiceResponseAsync(
this MethodDefinition method,
ScenarioDefinition[] scenarios,
IServiceAccount account,
AuthenicationCredentials credentials)
{
if (null == method)
throw new ArgumentNullException("method");
if (null == account)
throw new ArgumentNullException("account");
if (null == credentials)
throw new ArgumentNullException("credentials");
ValidationResults results = new ValidationResults();
if (scenarios.Length == 0)
{
// If no descenarios are defined for this method, add a new default scenario
scenarios = new ScenarioDefinition[] { new ScenarioDefinition { Description = "default-scenario", Enabled = true, MethodName = method.Identifier} };
results.AddResult("init", new ValidationMessage(null, "No scenarios were defined for method {0}. Will execute request as written.", method.Identifier), ValidationOutcome.None);
}
if (scenarios.Any() && !scenarios.Any(x => x.Enabled))
{
results.AddResult("init",
new ValidationWarning(ValidationErrorCode.AllScenariosDisabled, null, "All scenarios for method {0} were disabled.", method.Identifier),
ValidationOutcome.Skipped);
return results;
}
foreach (var scenario in scenarios.Where(x => x.Enabled))
{
try
{
await ValidateMethodWithScenarioAsync(method, scenario, account, credentials, results);
}
catch (Exception ex)
{
results.AddResult(
"validation",
new ValidationError(
ValidationErrorCode.ExceptionWhileValidatingMethod,
method.SourceFile.DisplayName,
ex.Message));
}
}
return results;
}
private static async Task ValidateMethodWithScenarioAsync(
MethodDefinition method,
ScenarioDefinition scenario,
IServiceAccount account,
AuthenicationCredentials credentials,
ValidationResults results)
{
if (null == method)
throw new ArgumentNullException("method");
if (null == scenario)
throw new ArgumentNullException("scenario");
if (null == account)
throw new ArgumentNullException("account");
if (null == credentials)
throw new ArgumentNullException("credentials");
if (null == results)
throw new ArgumentNullException("results");
var actionName = scenario.Description;
// Generate the tested request by "previewing" the request and executing
// all test-setup procedures
long startTicks = DateTimeOffset.UtcNow.Ticks;
var requestPreviewResult = await method.GenerateMethodRequestAsync(scenario, account.BaseUrl, credentials, method.SourceFile.Parent);
TimeSpan generateMethodDuration = new TimeSpan(DateTimeOffset.UtcNow.Ticks - startTicks);
// Check to see if an error occured building the request, and abort if so.
var generatorResults = results[actionName + " [test-setup requests]"];
generatorResults.AddResults(requestPreviewResult.Messages, requestPreviewResult.IsWarningOrError ? ValidationOutcome.Error : ValidationOutcome.Passed);
generatorResults.Duration = generateMethodDuration;
if (requestPreviewResult.IsWarningOrError)
{
return;
}
// We've done all the test-setup work, now we have the real request to make to the service
HttpRequest requestPreview = requestPreviewResult.Value;
results.AddResult(
actionName,
new ValidationMessage(null, "Generated Method HTTP Request:\r\n{0}", requestPreview.FullHttpText()));
HttpParser parser = new HttpParser();
HttpResponse expectedResponse = null;
if (!string.IsNullOrEmpty(method.ExpectedResponse))
{
expectedResponse = parser.ParseHttpResponse(method.ExpectedResponse);
}
// Execute the actual tested method (the result of the method preview call, which made the test-setup requests)
startTicks = DateTimeOffset.UtcNow.Ticks;
var actualResponse = await requestPreview.GetResponseAsync(account.BaseUrl);
TimeSpan actualMethodDuration = new TimeSpan(DateTimeOffset.UtcNow.Ticks - startTicks);
var requestResults = results[actionName];
if (actualResponse.RetryCount > 0)
{
requestResults.AddResults(
new ValidationError[]
{ new ValidationWarning(ValidationErrorCode.RequestWasRetried, null, "HTTP request was retried {0} times.", actualResponse.RetryCount) });
}
requestResults.AddResults(
new ValidationError[]
{ new ValidationMessage(null, "HTTP Response:\r\n{0}", actualResponse.FullText(false)) });
requestResults.Duration = actualMethodDuration;
// Perform validation on the method's actual response
ValidationError[] errors;
method.ValidateResponse(actualResponse, expectedResponse, scenario, out errors);
requestResults.AddResults(errors);
// TODO: If the method is defined as a long running operation, we need to go poll the status
// URL to make sure that the operation finished and the response type is valid.
if (errors.WereErrors())
results.SetOutcome(actionName, ValidationOutcome.Error);
else if (errors.WereWarnings())
results.SetOutcome(actionName, ValidationOutcome.Warning);
else
results.SetOutcome(actionName, ValidationOutcome.Passed);
}
private static async Task ValidateMethodWithScenarioAsync(
MethodDefinition method,
ScenarioDefinition scenario,
IServiceAccount account,
AuthenicationCredentials credentials,
ValidationResults results,
ValidationOptions options = null)
{
if (null == method)
{
throw new ArgumentNullException("method");
}
if (null == scenario)
{
throw new ArgumentNullException("scenario");
}
if (null == account)
{
throw new ArgumentNullException("account");
}
if (null == credentials)
{
throw new ArgumentNullException("credentials");
}
if (null == results)
{
throw new ArgumentNullException("results");
}
var actionName = scenario.Description;
// Check to see if the account + scenario scopes are aligned
string[] requiredScopes = method.RequiredScopes.Union(scenario.RequiredScopes).ToArray();
if (!account.Scopes.ProvidesScopes(requiredScopes, options.IgnoreRequiredScopes))
{
var missingScopes = from scope in requiredScopes where !account.Scopes.Contains(scope) select scope;
results.AddResult(actionName,
new ValidationWarning(ValidationErrorCode.RequiredScopesMissing,
null,
"Scenario was not run. Scopes required were not available: {0}", missingScopes.ComponentsJoinedByString(",")));
return;
}
// Generate the tested request by "previewing" the request and executing
// all test-setup procedures
long startTicks = DateTimeOffset.UtcNow.Ticks;
var requestPreviewResult = await method.GenerateMethodRequestAsync(scenario, account.BaseUrl, credentials, method.SourceFile.Parent);
TimeSpan generateMethodDuration = new TimeSpan(DateTimeOffset.UtcNow.Ticks - startTicks);
// Check to see if an error occured building the request, and abort if so.
var generatorResults = results[actionName /*+ " [test-setup requests]"*/];
generatorResults.AddResults(requestPreviewResult.Messages, requestPreviewResult.IsWarningOrError ? ValidationOutcome.Error : ValidationOutcome.Passed);
generatorResults.Duration = generateMethodDuration;
if (requestPreviewResult.IsWarningOrError)
{
return;
}
// We've done all the test-setup work, now we have the real request to make to the service
HttpRequest requestPreview = requestPreviewResult.Value;
results.AddResult(
actionName,
new ValidationMessage(null, "Generated Method HTTP Request:\r\n{0}", requestPreview.FullHttpText()));
HttpParser parser = new HttpParser();
HttpResponse expectedResponse = null;
if (!string.IsNullOrEmpty(method.ExpectedResponse))
{
expectedResponse = parser.ParseHttpResponse(method.ExpectedResponse);
}
// Execute the actual tested method (the result of the method preview call, which made the test-setup requests)
startTicks = DateTimeOffset.UtcNow.Ticks;
var actualResponse = await requestPreview.GetResponseAsync(account.BaseUrl);
TimeSpan actualMethodDuration = new TimeSpan(DateTimeOffset.UtcNow.Ticks - startTicks);
var requestResults = results[actionName];
if (actualResponse.RetryCount > 0)
{
requestResults.AddResults(
new ValidationError[]
{ new ValidationWarning(ValidationErrorCode.RequestWasRetried, null, "HTTP request was retried {0} times.", actualResponse.RetryCount) });
}
requestResults.AddResults(
new ValidationError[]
{ new ValidationMessage(null, "HTTP Response:\r\n{0}", actualResponse.FullText(false)) });
requestResults.Duration = actualMethodDuration;
// Perform validation on the method's actual response
ValidationError[] errors;
method.ValidateResponse(actualResponse, expectedResponse, scenario, out errors, options);
requestResults.AddResults(errors);
// TODO: If the method is defined as a long running operation, we need to go poll the status
// URL to make sure that the operation finished and the response type is valid.
if (errors.WereErrors())
{
results.SetOutcome(actionName, ValidationOutcome.Error);
}
else if (errors.WereWarnings())
{
results.SetOutcome(actionName, ValidationOutcome.Warning);
}
else
{
results.SetOutcome(actionName, ValidationOutcome.Passed);
}
}
internal static void AddAccessTokenToRequest(AuthenicationCredentials credentials, HttpRequest request)
{
credentials.AuthenticateRequest(request);
}
internal static void AddAccessTokenToRequest(AuthenicationCredentials credentials, HttpRequest request)
{
credentials.AuthenticateRequest(request);
}
/// <summary>
/// Take a scenario definition and convert the prototype request into a fully formed request. This includes appending
/// the base URL to the request URL, executing any test-setup requests, and replacing the placeholders in the prototype
/// request with proper values.
/// </summary>
/// <param name="scenario"></param>
/// <param name="baseUrl"></param>
/// <param name="credentials"></param>
/// <param name="documents"></param>
/// <returns></returns>
public async Task <ValidationResult <HttpRequest> > GenerateMethodRequestAsync(ScenarioDefinition scenario, string baseUrl, AuthenicationCredentials credentials, DocSet documents)
{
var parser = new HttpParser();
var request = parser.ParseHttpRequest(this.Request);
AddAccessTokenToRequest(credentials, request);
AddTestHeaderToRequest(scenario, request);
List <ValidationError> errors = new List <ValidationError>();
if (null != scenario)
{
var storedValuesForScenario = new Dictionary <string, string>();
if (null != scenario.TestSetupRequests)
{
foreach (var setupRequest in scenario.TestSetupRequests)
{
var result = await setupRequest.MakeSetupRequestAsync(baseUrl, credentials, storedValuesForScenario, documents, scenario);
errors.AddRange(result.Messages);
if (result.IsWarningOrError)
{
// If we can an error or warning back from a setup method, we fail the whole request.
return(new ValidationResult <HttpRequest>(null, errors));
}
}
}
try
{
var placeholderValues = scenario.RequestParameters.ToPlaceholderValuesArray(storedValuesForScenario);
request.RewriteRequestWithParameters(placeholderValues);
}
catch (Exception ex)
{
// Error when applying parameters to the request
errors.Add(
new ValidationError(
ValidationErrorCode.RewriteRequestFailure,
"GenerateMethodRequestAsync",
ex.Message));
return(new ValidationResult <HttpRequest>(null, errors));
}
if (scenario.StatusCodesToRetry != null)
{
request.RetryOnStatusCode =
(from status in scenario.StatusCodesToRetry select(System.Net.HttpStatusCode) status).ToList();
}
}
if (string.IsNullOrEmpty(request.Accept))
{
if (!string.IsNullOrEmpty(ValidationConfig.ODataMetadataLevel))
{
request.Accept = MimeTypeJson + "; " + ValidationConfig.ODataMetadataLevel;
}
else
{
request.Accept = MimeTypeJson;
}
}
return(new ValidationResult <HttpRequest>(request, errors));
}
/// <summary>
/// Take a scenario definition and convert the prototype request into a fully formed request. This includes appending
/// the base URL to the request URL, executing any test-setup requests, and replacing the placeholders in the prototype
/// request with proper values.
/// </summary>
/// <param name="scenario"></param>
/// <param name="baseUrl"></param>
/// <param name="credentials"></param>
/// <param name="documents"></param>
/// <returns></returns>
public async Task<ValidationResult<HttpRequest>> GenerateMethodRequestAsync(ScenarioDefinition scenario, string baseUrl, AuthenicationCredentials credentials, DocSet documents)
{
var parser = new HttpParser();
var request = parser.ParseHttpRequest(this.Request);
AddAccessTokenToRequest(credentials, request);
AddTestHeaderToRequest(scenario, request);
List<ValidationError> errors = new List<ValidationError>();
if (null != scenario)
{
var storedValuesForScenario = new Dictionary<string, string>();
if (null != scenario.TestSetupRequests)
{
foreach (var setupRequest in scenario.TestSetupRequests)
{
var result = await setupRequest.MakeSetupRequestAsync(baseUrl, credentials, storedValuesForScenario, documents, scenario);
errors.AddRange(result.Messages);
if (result.IsWarningOrError)
{
// If we can an error or warning back from a setup method, we fail the whole request.
return new ValidationResult<HttpRequest>(null, errors);
}
}
}
try
{
var placeholderValues = scenario.RequestParameters.ToPlaceholderValuesArray(storedValuesForScenario);
request.RewriteRequestWithParameters(placeholderValues);
}
catch (Exception ex)
{
// Error when applying parameters to the request
errors.Add(
new ValidationError(
ValidationErrorCode.RewriteRequestFailure,
"GenerateMethodRequestAsync",
ex.Message));
return new ValidationResult<HttpRequest>(null, errors);
}
if (scenario.StatusCodesToRetry != null)
{
request.RetryOnStatusCode =
(from status in scenario.StatusCodesToRetry select (System.Net.HttpStatusCode)status).ToList();
}
}
if (string.IsNullOrEmpty(request.Accept))
{
if (!string.IsNullOrEmpty(ValidationConfig.ODataMetadataLevel))
{
request.Accept = MimeTypeJson + "; " + ValidationConfig.ODataMetadataLevel;
}
else
{
request.Accept = MimeTypeJson;
}
}
return new ValidationResult<HttpRequest>(request, errors);
}