/// <summary>
/// Performs validation of a method (request/response) with a given service
/// target and zero or more test scenarios
/// </summary>
/// <param name="method"></param>
/// <param name="account"></param>
/// <param name="credentials"></param>
/// <returns></returns>
public static async Task <ValidationResults> ValidateServiceResponseAsync(
this MethodDefinition method,
ScenarioDefinition[] scenarios,
IServiceAccount account,
ValidationOptions options = null)
{
if (null == method)
{
throw new ArgumentNullException("method");
}
if (null == account)
{
throw new ArgumentNullException("account");
}
ValidationResults results = new ValidationResults();
if (scenarios.Length == 0)
{
// If no descenarios are defined for this method, add a new default scenario
scenarios = new ScenarioDefinition[] {
new ScenarioDefinition {
Description = "verbatim",
Enabled = true,
MethodName = method.Identifier,
RequiredScopes = method.RequiredScopes
}
};
// results.AddResult("init", new ValidationMessage(null, "No scenarios were defined for method {0}. Will request verbatim from docs.", method.Identifier), ValidationOutcome.None);
}
if (scenarios.Any() && !scenarios.Any(x => x.Enabled))
{
results.AddResult("init",
new ValidationWarning(ValidationErrorCode.AllScenariosDisabled, null, "All scenarios for method {0} were disabled.", method.Identifier),
ValidationOutcome.Skipped);
return(results);
}
foreach (var scenario in scenarios.Where(x => x.Enabled))
{
try
{
await ValidateMethodWithScenarioAsync(method, scenario, account, results, options);
}
catch (Exception ex)
{
results.AddResult(
"validation",
new ValidationError(
ValidationErrorCode.ExceptionWhileValidatingMethod,
method.SourceFile.DisplayName,
ex.Message));
}
}
return(results);
}
public PoorManControllerFactory(IServiceUser serviceUser, IServiceAccount serviceAccount)
{
if (serviceUser == null) throw new ArgumentNullException("serviceUser cannot be null");
if (serviceAccount == null) throw new ArgumentNullException("serviceAccount cannot be null");
_serviceAccount = serviceAccount;
_serviceUser = serviceUser;
}
/// <summary>
/// Performs validation of a method (request/response) with a given service
/// target and zero or more test scenarios
/// </summary>
/// <param name="method"></param>
/// <param name="account"></param>
/// <param name="credentials"></param>
/// <returns></returns>
public static async Task<ValidationResults> ValidateServiceResponseAsync(
this MethodDefinition method,
ScenarioDefinition[] scenarios,
IServiceAccount account,
ValidationOptions options = null)
{
if (null == method)
throw new ArgumentNullException("method");
if (null == account)
throw new ArgumentNullException("account");
ValidationResults results = new ValidationResults();
if (scenarios.Length == 0)
{
// If no descenarios are defined for this method, add a new default scenario
scenarios = new ScenarioDefinition[] {
new ScenarioDefinition {
Description = "verbatim",
Enabled = true,
MethodName = method.Identifier,
RequiredScopes = method.RequiredScopes
}
};
// results.AddResult("init", new ValidationMessage(null, "No scenarios were defined for method {0}. Will request verbatim from docs.", method.Identifier), ValidationOutcome.None);
}
if (scenarios.Any() && !scenarios.Any(x => x.Enabled))
{
results.AddResult("init",
new ValidationWarning(ValidationErrorCode.AllScenariosDisabled, null, "All scenarios for method {0} were disabled.", method.Identifier),
ValidationOutcome.Skipped);
return results;
}
foreach (var scenario in scenarios.Where(x => x.Enabled))
{
try
{
await ValidateMethodWithScenarioAsync(method, scenario, account, results, options);
}
catch (Exception ex)
{
results.AddResult(
"validation",
new ValidationError(
ValidationErrorCode.ExceptionWhileValidatingMethod,
method.SourceFile.DisplayName,
ex.Message));
}
}
return results;
}
/// <summary>
/// Write the results of a test to the output console.
/// </summary>
/// <param name="method"></param>
/// <param name="account"></param>
/// <param name="results"></param>
/// <param name="options"></param>
private static void PrintResultsToConsole(MethodDefinition method, IServiceAccount account, ValidationResults output, CheckServiceOptions options)
{
// Only allow one thread at a time to write to the console so we don't interleave our results.
lock (typeof(Program))
{
FancyConsole.WriteLine(
FancyConsole.ConsoleHeaderColor,
"Testing method {0} with account {1}",
method.Identifier,
account.Name);
foreach (var scenario in output.Results)
{
if (scenario.Errors.Count > 0)
{
FancyConsole.WriteLineIndented(
" ",
FancyConsole.ConsoleSubheaderColor,
"Scenario: {0}",
scenario.Name);
foreach (var message in scenario.Errors)
{
if (options.EnableVerboseOutput || message.IsWarningOrError)
{
FancyConsole.WriteLineIndented(
" ",
FancyConsole.ConsoleDefaultColor,
message.ErrorText);
}
}
if (options.SilenceWarnings && scenario.Outcome == ValidationOutcome.Warning)
{
scenario.Outcome = ValidationOutcome.Passed;
}
FancyConsole.WriteLineIndented(
" ",
scenario.Outcome.ConsoleColor(),
"Scenario finished with outcome: {0}. Duration: {1}",
scenario.Outcome,
scenario.Duration);
}
}
FancyConsole.WriteLineIndented(
" ",
output.OverallOutcome.ConsoleColor(),
"Method testing finished with overall outcome: {0}",
output.OverallOutcome);
FancyConsole.WriteLine();
}
}
/// <summary>
/// Breaks down the key/value string and adds to the request header
/// </summary>
/// <param name="headerKeyValueString"></param>
/// <returns></returns>
internal static void AddAdditionalHeadersToRequest(IServiceAccount account, HttpRequest request)
{
// parse the passed in addtional headers and add to request..format for headers should be <HeaderName>:<HeaderValue>
if (account.AdditionalHeaders != null && account.AdditionalHeaders.Length > 0)
{
foreach (string nameValueHeader in account.AdditionalHeaders)
{
string[] split = nameValueHeader.Split(new Char[] { ':' }, 2);
request.Headers.Add(split[0], split[1]);
}
}
}
/// <summary>
/// This method will adapt a request based on parameters for an account. It should be the last thing we do before sending
/// the request to the account.
/// </summary>
/// <param name="request"></param>
/// <param name="account"></param>
public void ModifyRequestForAccount(HttpRequest request, IServiceAccount account)
{
if (account.Transformations?.Request?.Actions?.Prefix == null)
{
return;
}
if (this.RequestMetadata.Target == TargetType.Action || this.RequestMetadata.Target == TargetType.Function)
{
// Add the ActionPrefix to the last path component of the URL
AddPrefixToLastUrlComponent(request, account.Transformations.Request.Actions.Prefix);
}
}
public static void Validate(IServiceAccount instance)
{
var userAccount = UserAccount.ParseAccountName(instance.ServiceAccount);
if (!userAccount.CheckPassword(instance.ServiceAccountPwd))
{
throw new EngineValidationException("Password Invalid");
}
try
{
SetLogonAsAServicePrivilege(userAccount);
}
catch (Exception)
{
throw new Exception($"Failed to enable the LogonAsAService privilege on {instance.ServiceAccount}");
}
}
public static void RewriteResponseBodyNamespaces(this HttpResponse response, IServiceAccount account, IssueLogger issues)
{
if (account.Transformations?.Response?.Properties == null)
{
return;
}
if (!response.IsMatchingContentType("application/json"))
{
return;
}
if (response.Body.Length == 0)
{
return;
}
var translatedBody = JsonRewriter.RewriteJsonProperties(response.Body, account.Transformations.Response.Properties, issues);
response.Body = translatedBody;
}
/// <summary>
/// Apply any namespace translations defined in the account to the parameters in the request body
/// </summary>
/// <param name="request"></param>
/// <param name="account"></param>
public static void RewriteRequestBodyNamespaces(this HttpRequest request, IServiceAccount account, IssueLogger issues)
{
if (account.Transformations?.Request?.Properties == null)
{
return;
}
if (request.IsMatchingContentType("application/json"))
{
var translatedBody = JsonRewriter.RewriteJsonProperties(request.Body, account.Transformations.Request.Properties, issues);
request.Body = translatedBody;
}
else if (request.IsMatchingContentType("multipart/related"))
{
var parsedBody = new MultipartMime.MultipartMimeContent(request.ContentType, request.Body);
var partsToRewrite = parsedBody.PartsWithContentType("application/json");
foreach (var part in partsToRewrite)
{
part.Body = JsonRewriter.RewriteJsonProperties(part.Body, account.Transformations.Request.Properties, issues);
}
request.Body = parsedBody.ToString();
}
}
public async Task <HttpResponse> GetResponseAsync(IServiceAccount account, int retryCount = 0)
{
var baseUrl = account.BaseUrl;
this.RewriteRequestBodyNamespaces(account);
var webRequest = this.PrepareHttpWebRequest(baseUrl);
this.StartTime = DateTimeOffset.UtcNow;
HttpResponse response = await HttpResponse.ResponseFromHttpWebResponseAsync(webRequest);
TimeSpan duration = DateTimeOffset.UtcNow.Subtract(this.StartTime);
response.RewriteResponseBodyNamespaces(account);
var logger = HttpRequest.HttpLogSession;
if (null != logger)
{
await logger.RecordSessionAsync(this, response, duration);
}
if (ShouldRetryRequest(response))
{
if (retryCount < ValidationConfig.RetryAttemptsOnServiceUnavailableResponse)
{
// Do "full jitter" back off
await BackoffHelper.Default.FullJitterBackoffDelayAsync(retryCount);
// Try the request again
return(await this.GetResponseAsync(account, retryCount + 1));
}
}
response.RetryCount = retryCount;
return(response);
}
internal static async Task LogMethodTestResults(Validation.MethodDefinition method, IServiceAccount account, Validation.ValidationResults results)
{
foreach (var scenario in results.Results)
{
if (scenario.Outcome == ValidationOutcome.None)
{
continue;
}
StringBuilder stdout = new StringBuilder();
string message = null;
if (scenario.Errors.Count > 0)
{
stdout.AppendFormat("Scenario: {0}", scenario.Name);
foreach (var error in scenario.Errors)
{
stdout.AppendLine(error.ErrorText);
if (error.IsError && message == null)
{
message = error.ErrorText.FirstLineOnly();
}
}
stdout.AppendFormat(
"Scenario finished with outcome: {0}. Duration: {1}",
scenario.Outcome,
scenario.Duration);
}
string testName = string.Format(
"{0}: {1} [{2}]",
method.Identifier,
scenario.Name,
account.Name);
string filename = method.SourceFile.DisplayName;
TestOutcome outcome = ToTestOutcome(scenario.Outcome);
await BuildWorkerApi.RecordTestAsync(
testName,
TestFrameworkName,
outcome : outcome,
durationInMilliseconds : (long)scenario.Duration.TotalMilliseconds,
errorMessage : message ?? scenario.Outcome.ToString(),
filename : filename,
stdOut : stdout.ToString());
}
}
/// <summary>
/// Take a scenario definition and convert the prototype request into a fully formed request. This includes appending
/// the base URL to the request URL, executing any test-setup requests, and replacing the placeholders in the prototype
/// request with proper values.
/// </summary>
/// <param name="scenario"></param>
/// <param name="documents"></param>
/// <param name="account"></param>
/// <returns></returns>
public async Task <ValidationResult <HttpRequest> > GenerateMethodRequestAsync(ScenarioDefinition scenario, DocSet documents, IServiceAccount account)
{
var parser = new HttpParser();
var request = parser.ParseHttpRequest(this.Request);
AddAccessTokenToRequest(account.CreateCredentials(), request);
AddTestHeaderToRequest(scenario, request);
AddAdditionalHeadersToRequest(account, request);
List <ValidationError> errors = new List <ValidationError>();
if (null != scenario)
{
var storedValuesForScenario = new Dictionary <string, string>();
if (null != scenario.TestSetupRequests)
{
foreach (var setupRequest in scenario.TestSetupRequests)
{
try
{
var result = await setupRequest.MakeSetupRequestAsync(storedValuesForScenario, documents, scenario, account);
errors.AddRange(result.Messages);
if (result.IsWarningOrError)
{
// If we can an error or warning back from a setup method, we fail the whole request.
return(new ValidationResult <HttpRequest>(null, errors));
}
}
catch (Exception ex)
{
return(new ValidationResult <HttpRequest>(null, new ValidationError(ValidationErrorCode.ConsolidatedError, null, "An exception occured while processing setup-requests: {0}", ex.Message)));
}
}
}
try
{
var placeholderValues = scenario.RequestParameters.ToPlaceholderValuesArray(storedValuesForScenario);
request.RewriteRequestWithParameters(placeholderValues);
}
catch (Exception ex)
{
// Error when applying parameters to the request
errors.Add(
new ValidationError(
ValidationErrorCode.RewriteRequestFailure,
"GenerateMethodRequestAsync",
ex.Message));
return(new ValidationResult <HttpRequest>(null, errors));
}
if (scenario.StatusCodesToRetry != null)
{
request.RetryOnStatusCode =
(from status in scenario.StatusCodesToRetry select(System.Net.HttpStatusCode) status).ToList();
}
}
if (string.IsNullOrEmpty(request.Accept))
{
if (!string.IsNullOrEmpty(ValidationConfig.ODataMetadataLevel))
{
request.Accept = MimeTypeJson + "; " + ValidationConfig.ODataMetadataLevel;
}
else
{
request.Accept = MimeTypeJson;
}
}
this.ModifyRequestForAccount(request, account);
return(new ValidationResult <HttpRequest>(request, errors));
}
internal static async Task LogMethodTestResults(Validation.MethodDefinition method, IServiceAccount account, Validation.ValidationResults results)
{
foreach (var scenario in results.Results)
{
if (scenario.Outcome == ValidationOutcome.None)
continue;
StringBuilder stdout = new StringBuilder();
string message = null;
if (scenario.Errors.Count > 0)
{
stdout.AppendFormat("Scenario: {0}", scenario.Name);
foreach (var error in scenario.Errors)
{
stdout.AppendLine(error.ErrorText);
if (error.IsError && message == null)
{
message = error.ErrorText.FirstLineOnly();
}
}
stdout.AppendFormat(
"Scenario finished with outcome: {0}. Duration: {1}",
scenario.Outcome,
scenario.Duration);
}
string testName = string.Format(
"{0}: {1} [{2}]",
method.Identifier,
scenario.Name,
account.Name);
string filename = method.SourceFile.DisplayName;
TestOutcome outcome = ToTestOutcome(scenario.Outcome);
await BuildWorkerApi.RecordTestAsync(
testName,
TestFrameworkName,
outcome: outcome,
durationInMilliseconds: (long)scenario.Duration.TotalMilliseconds,
errorMessage: message ?? scenario.Outcome.ToString(),
filename: filename,
stdOut: stdout.ToString());
}
}
/// <summary>
/// Make a call to the HttpRequest and populate the Value property of
/// every PlaceholderValue in Values based on the result.
/// </summary>
/// <param name="storedValues"></param>
/// <param name="documents"></param>
/// <param name="scenario"></param>
/// <param name="account"></param>
/// <param name="parentOutputValues"></param>
/// <returns></returns>
public async Task <ValidationResult <bool> > MakeSetupRequestAsync(
Dictionary <string, string> storedValues,
DocSet documents,
ScenarioDefinition scenario,
IServiceAccount account,
Dictionary <string, string> parentOutputValues = null)
{
// Copy the output values from parentOutputValues into our own
if (null != parentOutputValues)
{
foreach (var key in parentOutputValues.Keys)
{
this.OutputValues.Add(key, parentOutputValues[key]);
}
}
var errors = new List <ValidationError>();
if (!string.IsNullOrEmpty(this.CannedRequestName))
{
// We need to make a canned request setup request instead. Look up the canned request and then execute it, returning the results here.
var cannedRequest =
(from cr in documents.CannedRequests where cr.Name == this.CannedRequestName select cr)
.FirstOrDefault();
if (null == cannedRequest)
{
errors.Add(new ValidationError(ValidationErrorCode.InvalidRequestFormat, null, "Couldn't locate the canned-request named: {0}", this.CannedRequestName));
return(new ValidationResult <bool>(false, errors));
}
// Need to make a copy of the canned request here so that our state doesn't continue to pile up
var cannedRequestInstance = cannedRequest.CopyInstance();
return(await cannedRequestInstance.MakeSetupRequestAsync(storedValues, documents, scenario, account, this.OutputValues));
}
// Get the HttpRequest, either from MethodName or by parsing HttpRequest
HttpRequest request;
var issues = new IssueLogger();
try
{
request = this.GetHttpRequest(documents, account, issues);
}
catch (Exception ex)
{
errors.Add(new ValidationError(ValidationErrorCode.InvalidRequestFormat, null, "An error occured creating the http request: {0}", ex.Message));
return(new ValidationResult <bool>(false, errors));
}
MethodDefinition.AddTestHeaderToRequest(scenario, request);
MethodDefinition.AddAdditionalHeadersToRequest(account, request);
// If this is a canned request, we need to merge the parameters / placeholders here
var placeholderValues = this.RequestParameters.ToPlaceholderValuesArray(storedValues);
// Update the request with the parameters in request-parameters
try
{
request.RewriteRequestWithParameters(placeholderValues);
}
catch (Exception ex)
{
errors.Add(new ValidationError(ValidationErrorCode.ParameterParserError, SourceName, "Error rewriting the request with parameters from the scenario: {0}", ex.Message));
return(new ValidationResult <bool>(false, errors));
}
MethodDefinition.AddAccessTokenToRequest(account.CreateCredentials(), request);
errors.Add(new ValidationMessage(null, "Test-setup request:\n{0}", request.FullHttpText()));
try
{
var response = await request.GetResponseAsync(account, issues);
errors.AddRange(issues.Issues);
if (response.RetryCount > 0)
{
errors.Add(new ValidationWarning(ValidationErrorCode.RequestWasRetried, null, "HTTP request was retried {0} times.", response.RetryCount));
}
errors.Add(new ValidationMessage(null, "HTTP Response:\n{0}\n\n", response.FullText()));
// Check to see if this request is "successful" or not
if ((this.AllowedStatusCodes == null && response.WasSuccessful) ||
(this.AllowedStatusCodes != null && this.AllowedStatusCodes.Contains(response.StatusCode)))
{
string expectedContentType = (null != this.OutputValues) ? ExpectedResponseContentType(this.OutputValues.Values) : null;
// Check for content type mismatch
if (string.IsNullOrEmpty(response.ContentType) && expectedContentType != null)
{
return(new ValidationResult <bool>(false, new ValidationError(ValidationErrorCode.UnsupportedContentType, SourceName, "No Content-Type found for a non-204 response")));
}
// Load requested values into stored values
if (null != this.OutputValues)
{
foreach (var outputKey in this.OutputValues.Keys)
{
var source = this.OutputValues[outputKey];
storedValues[outputKey] = response.ValueForKeyedIdentifier(source);
}
}
return(new ValidationResult <bool>(!errors.Any(x => x.IsError), errors));
}
else
{
if ((this.AllowedStatusCodes != null && !this.AllowedStatusCodes.Contains(response.StatusCode)) || !response.WasSuccessful)
{
string expectedCodes = "200-299";
if (this.AllowedStatusCodes != null)
{
expectedCodes = this.AllowedStatusCodes.ComponentsJoinedByString(",");
}
errors.Add(new ValidationError(ValidationErrorCode.HttpStatusCodeDifferent, SourceName, "Http response status code {0} didn't match expected values: {1}", response.StatusCode, expectedCodes));
}
else
{
errors.Add(new ValidationError(ValidationErrorCode.HttpStatusCodeDifferent, SourceName, "Http response content type was invalid: {0}", response.ContentType));
}
return(new ValidationResult <bool>(false, errors));
}
}
catch (Exception ex)
{
errors.Add(new ValidationError(ValidationErrorCode.Unknown, SourceName, "Exception while making request: {0}", ex.Message));
return(new ValidationResult <bool>(false, errors));
}
}
public AppAccount(IAccount IAccount, IServiceAccount IServiceAccount)
{
_IAccount = IAccount;
_IServiceAccount = IServiceAccount;
}
private static void ConfigureAdditionalHeadersForAccount(CheckServiceOptions options, IServiceAccount account)
{
if (account.AdditionalHeaders != null && account.AdditionalHeaders.Length > 0)
{
// If the account needs additional headers, merge them in.
List <string> headers = new List <string>(account.AdditionalHeaders);
if (options.AdditionalHeaders != null)
{
headers.AddRange(options.AdditionalHeaders.Split('|'));
}
ValidationConfig.AdditionalHttpHeaders = headers.ToArray();
}
else if (options.AdditionalHeaders != null)
{
var headers = options.AdditionalHeaders.Split('|');
ValidationConfig.AdditionalHttpHeaders = headers.ToArray();
}
}
/// <summary>
/// Breaks down the key/value string and adds to the request header
/// </summary>
/// <param name="headerKeyValueString"></param>
/// <returns></returns>
internal static void AddAdditionalHeadersToRequest(IServiceAccount account, HttpRequest request)
{
// parse the passed in addtional headers and add to request..format for headers should be <HeaderName>:<HeaderValue>
if (account.AdditionalHeaders != null && account.AdditionalHeaders.Length > 0)
{
foreach (string nameValueHeader in account.AdditionalHeaders)
{
string[] split = nameValueHeader.Split(new Char[] { ':' }, 2);
request.Headers.Add(split[0], split[1]);
}
}
}
private static HttpRequest HttpRequestForCannedRequest(string requestName, DocSet docset, IServiceAccount account, IssueLogger issues)
{
var queryForRequest = from m in docset.CannedRequests
where m.Name == requestName
select m;
var foundRequest = queryForRequest.FirstOrDefault();
if (null == foundRequest)
{
throw new Exception(string.Format("Failed to find canned response {0} in the docset.", requestName));
}
return(GetHttpRequest(foundRequest, docset, account, issues));
}
private static HttpRequest LookupHttpRequestForMethod(string methodName, DocSet docset, IServiceAccount account, IssueLogger issues)
{
var queryForMethod = from m in docset.Methods
where m.Identifier == methodName
select m;
var foundMethod = queryForMethod.FirstOrDefault();
if (null == foundMethod)
{
throw new Exception(string.Format("Failed to locate method {0} in the docset.", methodName));
}
var request = ParseHttpRequest(foundMethod.Request, issues);
foundMethod.ModifyRequestForAccount(request, account);
return(request);
}
/// <summary>
/// Locate the Http.HttpRequest instance for this request definition either by
/// parsing the RawHttpRequest or resolving MethodName into a request.
/// </summary>
/// <param name="definition"></param>
/// <param name="documents"></param>
/// <returns></returns>
public static HttpRequest GetHttpRequest(this BasicRequestDefinition definition, DocSet documents, IServiceAccount account, IssueLogger issues)
{
HttpRequest foundRequest = null;
if (!string.IsNullOrEmpty(definition.RawHttpRequest))
{
foundRequest = ParseHttpRequest(definition.RawHttpRequest, issues);
}
else if (!string.IsNullOrEmpty(definition.MethodName))
{
foundRequest = LookupHttpRequestForMethod(definition.MethodName, documents, account, issues);
}
else if (!string.IsNullOrEmpty(definition.CannedRequestName))
{
foundRequest = HttpRequestForCannedRequest(definition.CannedRequestName, documents, account, issues);
}
return(foundRequest);
}
/// <summary>
/// Take a scenario definition and convert the prototype request into a fully formed request. This includes appending
/// the base URL to the request URL, executing any test-setup requests, and replacing the placeholders in the prototype
/// request with proper values.
/// </summary>
/// <param name="scenario"></param>
/// <param name="documents"></param>
/// <param name="account"></param>
/// <returns></returns>
public async Task<ValidationResult<HttpRequest>> GenerateMethodRequestAsync(ScenarioDefinition scenario, DocSet documents, IServiceAccount account)
{
var parser = new HttpParser();
var request = parser.ParseHttpRequest(this.Request);
AddAccessTokenToRequest(account.CreateCredentials(), request);
AddTestHeaderToRequest(scenario, request);
AddAdditionalHeadersToRequest(account, request);
List<ValidationError> errors = new List<ValidationError>();
if (null != scenario)
{
var storedValuesForScenario = new Dictionary<string, string>();
if (null != scenario.TestSetupRequests)
{
foreach (var setupRequest in scenario.TestSetupRequests)
{
var result = await setupRequest.MakeSetupRequestAsync(storedValuesForScenario, documents, scenario, account);
errors.AddRange(result.Messages);
if (result.IsWarningOrError)
{
// If we can an error or warning back from a setup method, we fail the whole request.
return new ValidationResult<HttpRequest>(null, errors);
}
}
}
try
{
var placeholderValues = scenario.RequestParameters.ToPlaceholderValuesArray(storedValuesForScenario);
request.RewriteRequestWithParameters(placeholderValues);
}
catch (Exception ex)
{
// Error when applying parameters to the request
errors.Add(
new ValidationError(
ValidationErrorCode.RewriteRequestFailure,
"GenerateMethodRequestAsync",
ex.Message));
return new ValidationResult<HttpRequest>(null, errors);
}
if (scenario.StatusCodesToRetry != null)
{
request.RetryOnStatusCode =
(from status in scenario.StatusCodesToRetry select (System.Net.HttpStatusCode)status).ToList();
}
}
if (string.IsNullOrEmpty(request.Accept))
{
if (!string.IsNullOrEmpty(ValidationConfig.ODataMetadataLevel))
{
request.Accept = MimeTypeJson + "; " + ValidationConfig.ODataMetadataLevel;
}
else
{
request.Accept = MimeTypeJson;
}
}
return new ValidationResult<HttpRequest>(request, errors);
}
private static async Task ValidateMethodWithScenarioAsync(
MethodDefinition method,
ScenarioDefinition scenario,
IServiceAccount account,
AuthenicationCredentials credentials,
ValidationResults results,
ValidationOptions options = null)
{
if (null == method)
{
throw new ArgumentNullException("method");
}
if (null == scenario)
{
throw new ArgumentNullException("scenario");
}
if (null == account)
{
throw new ArgumentNullException("account");
}
if (null == credentials)
{
throw new ArgumentNullException("credentials");
}
if (null == results)
{
throw new ArgumentNullException("results");
}
var actionName = scenario.Description;
// Check to see if the account + scenario scopes are aligned
string[] requiredScopes = method.RequiredScopes.Union(scenario.RequiredScopes).ToArray();
if (!account.Scopes.ProvidesScopes(requiredScopes, options.IgnoreRequiredScopes))
{
var missingScopes = from scope in requiredScopes where !account.Scopes.Contains(scope) select scope;
results.AddResult(actionName,
new ValidationWarning(ValidationErrorCode.RequiredScopesMissing,
null,
"Scenario was not run. Scopes required were not available: {0}", missingScopes.ComponentsJoinedByString(",")));
return;
}
// Generate the tested request by "previewing" the request and executing
// all test-setup procedures
long startTicks = DateTimeOffset.UtcNow.Ticks;
var requestPreviewResult = await method.GenerateMethodRequestAsync(scenario, account.BaseUrl, credentials, method.SourceFile.Parent);
TimeSpan generateMethodDuration = new TimeSpan(DateTimeOffset.UtcNow.Ticks - startTicks);
// Check to see if an error occured building the request, and abort if so.
var generatorResults = results[actionName /*+ " [test-setup requests]"*/];
generatorResults.AddResults(requestPreviewResult.Messages, requestPreviewResult.IsWarningOrError ? ValidationOutcome.Error : ValidationOutcome.Passed);
generatorResults.Duration = generateMethodDuration;
if (requestPreviewResult.IsWarningOrError)
{
return;
}
// We've done all the test-setup work, now we have the real request to make to the service
HttpRequest requestPreview = requestPreviewResult.Value;
results.AddResult(
actionName,
new ValidationMessage(null, "Generated Method HTTP Request:\r\n{0}", requestPreview.FullHttpText()));
HttpParser parser = new HttpParser();
HttpResponse expectedResponse = null;
if (!string.IsNullOrEmpty(method.ExpectedResponse))
{
expectedResponse = parser.ParseHttpResponse(method.ExpectedResponse);
}
// Execute the actual tested method (the result of the method preview call, which made the test-setup requests)
startTicks = DateTimeOffset.UtcNow.Ticks;
var actualResponse = await requestPreview.GetResponseAsync(account.BaseUrl);
TimeSpan actualMethodDuration = new TimeSpan(DateTimeOffset.UtcNow.Ticks - startTicks);
var requestResults = results[actionName];
if (actualResponse.RetryCount > 0)
{
requestResults.AddResults(
new ValidationError[]
{ new ValidationWarning(ValidationErrorCode.RequestWasRetried, null, "HTTP request was retried {0} times.", actualResponse.RetryCount) });
}
requestResults.AddResults(
new ValidationError[]
{ new ValidationMessage(null, "HTTP Response:\r\n{0}", actualResponse.FullText(false)) });
requestResults.Duration = actualMethodDuration;
// Perform validation on the method's actual response
ValidationError[] errors;
method.ValidateResponse(actualResponse, expectedResponse, scenario, out errors, options);
requestResults.AddResults(errors);
// TODO: If the method is defined as a long running operation, we need to go poll the status
// URL to make sure that the operation finished and the response type is valid.
if (errors.WereErrors())
{
results.SetOutcome(actionName, ValidationOutcome.Error);
}
else if (errors.WereWarnings())
{
results.SetOutcome(actionName, ValidationOutcome.Warning);
}
else
{
results.SetOutcome(actionName, ValidationOutcome.Passed);
}
}
public AccountController(IServiceUser serviceUser, IServiceAccount serviceAccount)
{
if (serviceAccount == null) throw new ArgumentNullException("serviceAccount cannot be null");
if (serviceUser == null) throw new ArgumentNullException("serviceUser cannot be null");
_serviceUser = serviceUser;
_serviceAccount = serviceAccount;
}
/// <summary>
/// Execute the provided methods on the given account.
/// </summary>
/// <param name="options"></param>
/// <param name="account"></param>
/// <param name="methods"></param>
/// <param name="docset"></param>
/// <returns>True if the methods all passed, false if there were failures.</returns>
private static async Task <bool> CheckMethodsForAccountAsync(CheckServiceOptions options, IServiceAccount account, MethodDefinition[] methods, DocSet docset)
{
//CheckResults results = new CheckResults();
ConfigureAdditionalHeadersForAccount(options, account);
string testNamePrefix = account.Name.ToLower() + ": ";
FancyConsole.WriteLine(FancyConsole.ConsoleHeaderColor, "Testing with account: {0}", account.Name);
FancyConsole.WriteLine(FancyConsole.ConsoleCodeColor, "Preparing authentication for requests...", account.Name);
try
{
await account.PrepareForRequestAsync();
}
catch (Exception ex)
{
RecordError(ex.Message);
return(false);
}
AuthenicationCredentials credentials = account.CreateCredentials();
int concurrentTasks = options.ParallelTests ? ParallelTaskCount : 1;
CheckResults docSetResults = new CheckResults();
await ForEachAsync(methods, concurrentTasks, async method =>
{
FancyConsole.WriteLine(
FancyConsole.ConsoleCodeColor,
"Running validation for method: {0}",
method.Identifier);
ScenarioDefinition[] scenarios = docset.TestScenarios.ScenariosForMethod(method);
ValidationResults results = await method.ValidateServiceResponseAsync(scenarios, account, credentials);
PrintResultsToConsole(method, account, results, options);
await TestReport.LogMethodTestResults(method, account, results);
docSetResults.RecordResults(results, options);
if (concurrentTasks == 1)
{
AddPause(options);
}
});
if (options.IgnoreWarnings || options.SilenceWarnings)
{
// Remove the warning flag from the outcomes
docSetResults.ConvertWarningsToSuccess();
}
docSetResults.PrintToConsole();
bool hadWarnings = docSetResults.WarningCount > 0;
bool hadErrors = docSetResults.FailureCount > 0;
return(!(hadErrors | hadWarnings));
}
/// <summary>
/// Make a call to the HttpRequest and populate the Value property of
/// every PlaceholderValue in Values based on the result.
/// </summary>
/// <param name="storedValues"></param>
/// <param name="documents"></param>
/// <param name="scenario"></param>
/// <param name="account"></param>
/// <param name="parentOutputValues"></param>
/// <returns></returns>
public async Task<ValidationResult<bool>> MakeSetupRequestAsync(Dictionary<string, string> storedValues, DocSet documents, ScenarioDefinition scenario, IServiceAccount account, Dictionary<string, string> parentOutputValues = null)
{
// Copy the output values from parentOutputValues into our own
if (null != parentOutputValues)
{
foreach (var key in parentOutputValues.Keys)
{
this.OutputValues.Add(key, parentOutputValues[key]);
}
}
var errors = new List<ValidationError>();
if (!string.IsNullOrEmpty(this.CannedRequestName))
{
// We need to make a canned request setup request instead. Look up the canned request and then execute it, returning the results here.
var cannedRequest =
(from cr in documents.CannedRequests where cr.Name == this.CannedRequestName select cr)
.FirstOrDefault();
if (null == cannedRequest)
{
errors.Add(new ValidationError(ValidationErrorCode.InvalidRequestFormat, null, "Couldn't locate the canned-request named: {0}", this.CannedRequestName));
return new ValidationResult<bool>(false, errors);
}
// Need to make a copy of the canned request here so that our state doesn't continue to pile up
var cannedRequestInstance = cannedRequest.CopyInstance();
return await cannedRequestInstance.MakeSetupRequestAsync(storedValues, documents, scenario, account, this.OutputValues);
}
// Get the HttpRequest, either from MethodName or by parsing HttpRequest
HttpRequest request;
try
{
request = this.GetHttpRequest(documents);
}
catch (Exception ex)
{
errors.Add(new ValidationError(ValidationErrorCode.InvalidRequestFormat, null, "An error occured creating the http request: {0}", ex.Message));
return new ValidationResult<bool>(false, errors);
}
MethodDefinition.AddTestHeaderToRequest(scenario, request);
MethodDefinition.AddAdditionalHeadersToRequest(account, request);
// If this is a canned request, we need to merge the parameters / placeholders here
var placeholderValues = this.RequestParameters.ToPlaceholderValuesArray(storedValues);
// Update the request with the parameters in request-parameters
try
{
request.RewriteRequestWithParameters(placeholderValues);
}
catch (Exception ex)
{
errors.Add(new ValidationError(ValidationErrorCode.ParameterParserError, SourceName, "Error rewriting the request with parameters from the scenario: {0}", ex.Message));
return new ValidationResult<bool>(false, errors);
}
MethodDefinition.AddAccessTokenToRequest(account.CreateCredentials(), request);
errors.Add(new ValidationMessage(null, "Test-setup request:\n{0}", request.FullHttpText()));
try
{
var response = await request.GetResponseAsync(account.BaseUrl);
if (response.RetryCount > 0)
{
errors.Add(new ValidationWarning(ValidationErrorCode.RequestWasRetried, null, "HTTP request was retried {0} times.", response.RetryCount));
}
errors.Add(new ValidationMessage(null, "HTTP Response:\n{0}\n\n", response.FullText()));
// Check to see if this request is "successful" or not
if ( (this.AllowedStatusCodes == null && response.WasSuccessful) ||
(this.AllowedStatusCodes != null && this.AllowedStatusCodes.Contains(response.StatusCode)))
{
string expectedContentType = (null != this.OutputValues) ? ExpectedResponseContentType(this.OutputValues.Values) : null;
// Check for content type mismatch
if (string.IsNullOrEmpty(response.ContentType) && expectedContentType != null)
{
return new ValidationResult<bool>(false, new ValidationError(ValidationErrorCode.UnsupportedContentType, SourceName, "No Content-Type found for a non-204 response"));
}
// Load requested values into stored values
if (null != this.OutputValues)
{
foreach (var outputKey in this.OutputValues.Keys)
{
var source = this.OutputValues[outputKey];
storedValues[outputKey] = response.ValueForKeyedIdentifier(source);
}
}
return new ValidationResult<bool>(!errors.Any(x => x.IsError), errors);
}
else
{
if ((this.AllowedStatusCodes != null && !this.AllowedStatusCodes.Contains(response.StatusCode)) || !response.WasSuccessful)
{
string expectedCodes = "200-299";
if (this.AllowedStatusCodes != null)
expectedCodes = this.AllowedStatusCodes.ComponentsJoinedByString(",");
errors.Add(new ValidationError(ValidationErrorCode.HttpStatusCodeDifferent, SourceName, "Http response status code {0} didn't match expected values: {1}", response.StatusCode, expectedCodes));
}
else
{
errors.Add(new ValidationError(ValidationErrorCode.HttpStatusCodeDifferent, SourceName, "Http response content type was invalid: {0}", response.ContentType));
}
return new ValidationResult<bool>(false, errors);
}
}
catch (Exception ex)
{
errors.Add(new ValidationError(ValidationErrorCode.Unknown, SourceName, "Exception while making request: {0}", ex.Message));
return new ValidationResult<bool>(false, errors);
}
}
public TransactionController(IServiceTransaction services, IEventBus bus, IServiceAccount servicesAccount)
{
_services = services;
_bus = bus;
_servicesAccount = servicesAccount;
}
private static async Task ValidateMethodWithScenarioAsync(
MethodDefinition method,
ScenarioDefinition scenario,
IServiceAccount account,
AuthenicationCredentials credentials,
ValidationResults results)
{
if (null == method)
throw new ArgumentNullException("method");
if (null == scenario)
throw new ArgumentNullException("scenario");
if (null == account)
throw new ArgumentNullException("account");
if (null == credentials)
throw new ArgumentNullException("credentials");
if (null == results)
throw new ArgumentNullException("results");
var actionName = scenario.Description;
// Generate the tested request by "previewing" the request and executing
// all test-setup procedures
long startTicks = DateTimeOffset.UtcNow.Ticks;
var requestPreviewResult = await method.GenerateMethodRequestAsync(scenario, account.BaseUrl, credentials, method.SourceFile.Parent);
TimeSpan generateMethodDuration = new TimeSpan(DateTimeOffset.UtcNow.Ticks - startTicks);
// Check to see if an error occured building the request, and abort if so.
var generatorResults = results[actionName + " [test-setup requests]"];
generatorResults.AddResults(requestPreviewResult.Messages, requestPreviewResult.IsWarningOrError ? ValidationOutcome.Error : ValidationOutcome.Passed);
generatorResults.Duration = generateMethodDuration;
if (requestPreviewResult.IsWarningOrError)
{
return;
}
// We've done all the test-setup work, now we have the real request to make to the service
HttpRequest requestPreview = requestPreviewResult.Value;
results.AddResult(
actionName,
new ValidationMessage(null, "Generated Method HTTP Request:\r\n{0}", requestPreview.FullHttpText()));
HttpParser parser = new HttpParser();
HttpResponse expectedResponse = null;
if (!string.IsNullOrEmpty(method.ExpectedResponse))
{
expectedResponse = parser.ParseHttpResponse(method.ExpectedResponse);
}
// Execute the actual tested method (the result of the method preview call, which made the test-setup requests)
startTicks = DateTimeOffset.UtcNow.Ticks;
var actualResponse = await requestPreview.GetResponseAsync(account.BaseUrl);
TimeSpan actualMethodDuration = new TimeSpan(DateTimeOffset.UtcNow.Ticks - startTicks);
var requestResults = results[actionName];
if (actualResponse.RetryCount > 0)
{
requestResults.AddResults(
new ValidationError[]
{ new ValidationWarning(ValidationErrorCode.RequestWasRetried, null, "HTTP request was retried {0} times.", actualResponse.RetryCount) });
}
requestResults.AddResults(
new ValidationError[]
{ new ValidationMessage(null, "HTTP Response:\r\n{0}", actualResponse.FullText(false)) });
requestResults.Duration = actualMethodDuration;
// Perform validation on the method's actual response
ValidationError[] errors;
method.ValidateResponse(actualResponse, expectedResponse, scenario, out errors);
requestResults.AddResults(errors);
// TODO: If the method is defined as a long running operation, we need to go poll the status
// URL to make sure that the operation finished and the response type is valid.
if (errors.WereErrors())
results.SetOutcome(actionName, ValidationOutcome.Error);
else if (errors.WereWarnings())
results.SetOutcome(actionName, ValidationOutcome.Warning);
else
results.SetOutcome(actionName, ValidationOutcome.Passed);
}
public AccountController(IServiceAccount services)
{
_services = services;
}
