/// <summary>
/// Performs validation of a method (request/response) with a given service
/// target and zero or more test scenarios
/// </summary>
/// <param name="method"></param>
/// <param name="account"></param>
/// <param name="credentials"></param>
/// <returns></returns>
public static async Task<ValidationResults> ValidateServiceResponseAsync(
this MethodDefinition method,
ScenarioDefinition[] scenarios,
IServiceAccount account,
ValidationOptions options = null)
{
if (null == method)
throw new ArgumentNullException("method");
if (null == account)
throw new ArgumentNullException("account");
ValidationResults results = new ValidationResults();
if (scenarios.Length == 0)
{
// If no descenarios are defined for this method, add a new default scenario
scenarios = new ScenarioDefinition[] {
new ScenarioDefinition {
Description = "verbatim",
Enabled = true,
MethodName = method.Identifier,
RequiredScopes = method.RequiredScopes
}
};
// results.AddResult("init", new ValidationMessage(null, "No scenarios were defined for method {0}. Will request verbatim from docs.", method.Identifier), ValidationOutcome.None);
}
if (scenarios.Any() && !scenarios.Any(x => x.Enabled))
{
results.AddResult("init",
new ValidationWarning(ValidationErrorCode.AllScenariosDisabled, null, "All scenarios for method {0} were disabled.", method.Identifier),
ValidationOutcome.Skipped);
return results;
}
foreach (var scenario in scenarios.Where(x => x.Enabled))
{
try
{
await ValidateMethodWithScenarioAsync(method, scenario, account, results, options);
}
catch (Exception ex)
{
results.AddResult(
"validation",
new ValidationError(
ValidationErrorCode.ExceptionWhileValidatingMethod,
method.SourceFile.DisplayName,
ex.Message));
}
}
return results;
}
public ValidationOptions(ValidationOptions options)
{
if (null == options)
return;
this.AllowTruncatedResponses = options.AllowTruncatedResponses;
this.RequiredPropertyNames = options.RequiredPropertyNames;
this.CollectionPropertyName = options.CollectionPropertyName;
this.ExpectedJsonSchema = options.ExpectedJsonSchema;
this.NullablePropertyNames = options.NullablePropertyNames;
this.RelaxedStringValidation = options.RelaxedStringValidation;
}
/// <summary>
/// Examines input json string to ensure that it compiles with the JsonSchema definition. Any errors in the
/// validation of the schema are returned via the errors out parameter.
/// </summary>
/// <param name="schema">Schemas definition used as a reference.</param>
/// <param name="inputJson">Input json example to be validated</param>
/// <param name="errors">Out parameter that provides any errors, warnings, or messages that were generated</param>
/// <param name="expectedJson"></param>
/// <returns></returns>
public bool ValidateJsonCompilesWithSchema(JsonSchema schema, JsonExample inputJson, out ValidationError[] errors, JsonExample expectedJson = null, ValidationOptions options = null)
{
if (null == schema)
throw new ArgumentNullException("schema");
if (null == inputJson)
throw new ArgumentNullException("inputJson");
string collectionPropertyName = "value";
if (null != inputJson.Annotation && null != inputJson.Annotation.CollectionPropertyName)
{
collectionPropertyName = inputJson.Annotation.CollectionPropertyName;
}
// If we didn't get an options, create a new one with some defaults provided by the annotation
options = options ?? new ValidationOptions();
options.AllowTruncatedResponses = (inputJson.Annotation ?? new CodeBlockAnnotation()).TruncatedResult;
options.CollectionPropertyName = collectionPropertyName;
return schema.ValidateJson(inputJson, out errors, this.registeredSchema, options, expectedJson);
}
/// <summary>
/// Validate the input json against the defined scehma when the instance was created.
/// </summary>
/// <param name="jsonInput">Input json to validate against schema</param>
/// <param name="errors">Array of errors if the validation fails</param>
/// <param name="otherSchemas"></param>
/// <param name="options"></param>
/// <param name="expectedJson"></param>
/// <returns>True if validation was successful, otherwise false.</returns>
public bool ValidateJson(JsonExample jsonInput, out ValidationError[] errors, Dictionary <string, JsonSchema> otherSchemas, ValidationOptions options, JsonExample expectedJson = null)
{
JContainer obj;
try
{
var settings = new JsonSerializerSettings {
DateParseHandling = DateParseHandling.None, NullValueHandling = NullValueHandling.Include, DefaultValueHandling = DefaultValueHandling.Include
};
obj = (JContainer)JsonConvert.DeserializeObject(jsonInput.JsonData, settings);
}
catch (Exception ex)
{
errors = new ValidationError[] { new ValidationError(ValidationErrorCode.JsonParserException, null, "Failed to parse json string: {0}. Json: {1}", ex.Message, jsonInput.JsonData) };
return(false);
}
var annotation = jsonInput.Annotation ?? new CodeBlockAnnotation();
List <ValidationError> detectedErrors = new List <ValidationError>();
bool expectErrorObject = (jsonInput.Annotation != null) && jsonInput.Annotation.ExpectError;
// Check for an error response
dynamic errorObject = obj["error"];
if (null != errorObject && !expectErrorObject)
{
string code = errorObject.code;
string message = errorObject.message;
detectedErrors.Clear();
detectedErrors.Add(new ValidationError(ValidationErrorCode.JsonErrorObject, null, "Error response received. Code: {0}, Message: {1}", code, message));
errors = detectedErrors.ToArray();
return(false);
}
else if (expectErrorObject && null == errorObject)
{
detectedErrors.Clear();
detectedErrors.Add(new ValidationError(ValidationErrorCode.JsonErrorObjectExpected, null, "Expected an error object response, but didn't receive one."));
errors = detectedErrors.ToArray();
return(false);
}
// Check to see if this is a "collection" instance
if (null != annotation && annotation.IsCollection)
{
this.ValidateCollectionObject(obj, annotation, otherSchemas, options.CollectionPropertyName, detectedErrors, options);
}
// otherwise verify the object matches this schema
else
{
options = options ?? new ValidationOptions(annotation);
if (null != expectedJson)
{
var expectedJsonSchema = new JsonSchema(expectedJson.JsonData, expectedJson.Annotation);
options.ExpectedJsonSchema = expectedJsonSchema;
options.RequiredPropertyNames = expectedJsonSchema.ExpectedProperties.Keys.ToArray();
}
this.ValidateContainerObject(obj, options, otherSchemas, detectedErrors);
}
errors = detectedErrors.ToArray();
return(detectedErrors.Count == 0);
}
private void ValidateCollectionObject(JContainer obj, CodeBlockAnnotation annotation, Dictionary <string, JsonSchema> otherSchemas, string collectionPropertyName, List <ValidationError> detectedErrors, ValidationOptions options)
{
// TODO: also validate additional properties on the collection, like nextDataLink
var collection = obj[collectionPropertyName];
if (null == collection)
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.MissingCollectionProperty, null, "Failed to locate collection property '{0}' in response.", collectionPropertyName));
}
else
{
var collectionMembers = obj[collectionPropertyName];
if (!collectionMembers.Any())
{
if (!annotation.IsEmpty)
{
detectedErrors.Add(
new ValidationWarning(
ValidationErrorCode.CollectionArrayEmpty,
null,
"Property contained an empty array that was not validated: {0}",
collectionPropertyName));
}
}
else if (annotation.IsEmpty)
{
detectedErrors.Add(
new ValidationWarning(
ValidationErrorCode.CollectionArrayNotEmpty,
null,
"Property contained a non-empty array that was expected to be empty: {0}",
collectionPropertyName));
}
foreach (var jToken in collectionMembers)
{
var container = jToken as JContainer;
if (null != container)
{
List <ValidationError> containerErrors = new List <ValidationError>();
var deeperOptions = new ValidationOptions(options)
{
AllowTruncatedResponses = annotation.TruncatedResult
};
this.ValidateContainerObject(
container,
deeperOptions,
otherSchemas,
containerErrors);
detectedErrors.AddUniqueErrors(containerErrors);
}
}
}
}
/// <summary>
/// Verify that a property from the json-to-validate matches something in our schema
/// </summary>
/// <param name="inputProperty"></param>
/// <param name="schemas"></param>
/// <param name="detectedErrors"></param>
/// <param name="options"></param>
/// <returns></returns>
private PropertyValidationOutcome ValidateProperty(JsonProperty inputProperty, Dictionary<string, JsonSchema> schemas, List<ValidationError> detectedErrors, ValidationOptions options)
{
if (this.ExpectedProperties.ContainsKey(inputProperty.Name))
{
// The property was expected to be found in this schema! Yay.
var schemaPropertyDef = this.ExpectedProperties[inputProperty.Name];
// Check for simple value types first
if (this.SimpleValueTypes(schemaPropertyDef.Type, inputProperty.Type) && this.AllFalse(schemaPropertyDef.IsArray, inputProperty.IsArray))
{
if (schemaPropertyDef.Type == inputProperty.Type && inputProperty.Type != JsonDataType.String)
{
return PropertyValidationOutcome.Ok;
}
else if (schemaPropertyDef.Type == inputProperty.Type && inputProperty.Type == JsonDataType.String)
{
// Perform extra validation to see if the string is the right format (iso date, enum value, url, or just a string)
if (null == options || options.RelaxedStringValidation)
return PropertyValidationOutcome.Ok;
return ValidateStringFormat(schemaPropertyDef, inputProperty, detectedErrors);
}
else
{
// Type of the inputProperty is mismatched from the expected value.
detectedErrors.Add(new ValidationError(ValidationErrorCode.ExpectedTypeDifferent, null, "Expected type {0} but was instead {1}: {2}", schemaPropertyDef.Type, inputProperty.Type, inputProperty.Name));
return PropertyValidationOutcome.InvalidType;
}
}
else if (null == inputProperty.OriginalValue)
{
if (null != this.NullableProperties && !this.NullableProperties.Contains(schemaPropertyDef.Name))
{
detectedErrors.Add(new ValidationWarning(ValidationErrorCode.NullPropertyValue, null, "Non-nullable property {0} had a null value in the response. Expected {1}.", schemaPropertyDef.Name, schemaPropertyDef.Type));
}
return PropertyValidationOutcome.Ok;
}
else if (schemaPropertyDef.IsArray || inputProperty.IsArray)
{
// Check for an array
if (schemaPropertyDef.IsArray && !inputProperty.IsArray)
{
// Expected an array, but didn't get one
detectedErrors.Add(new ValidationError(ValidationErrorCode.ExpectedArrayValue, null, "Expected an array but property was not an array: {0}", inputProperty.Name));
return PropertyValidationOutcome.InvalidType;
}
else if (!schemaPropertyDef.IsArray && inputProperty.IsArray)
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.ExpectedNonArrayValue, null, "Expected a value of type {0} but property was an array: {1}", schemaPropertyDef.Type, inputProperty.Name));
return PropertyValidationOutcome.InvalidType;
}
return this.ValidateArrayProperty(inputProperty, schemas, detectedErrors, options);
}
else if (schemaPropertyDef.Type == JsonDataType.ODataType && (inputProperty.Type == JsonDataType.Object || inputProperty.Type == JsonDataType.ODataType))
{
// Compare the ODataType schema to the custom schema
if (!schemas.ContainsKey(schemaPropertyDef.ODataTypeName))
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.ResourceTypeNotFound, null, "Missing resource: resource {0} was not found (property name '{1}').", schemaPropertyDef.ODataTypeName, inputProperty.Name));
return PropertyValidationOutcome.MissingResourceType;
}
else if (inputProperty.Type == JsonDataType.Object)
{
var odataSchema = schemas[schemaPropertyDef.ODataTypeName];
ValidationError[] odataErrors;
if (null != inputProperty.CustomMembers && !odataSchema.ValidateCustomObject(inputProperty.CustomMembers.Values.ToArray(), out odataErrors, schemas, options))
{
var propertyError = ValidationError.NewConsolidatedError(ValidationErrorCode.ConsolidatedError, odataErrors, "Schema validation failed on property '{0}' ['{1}']", inputProperty.Name, odataSchema.ResourceName);
detectedErrors.Add(propertyError);
return PropertyValidationOutcome.InvalidType;
}
else if (null == inputProperty.CustomMembers)
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.NoCustomMembersFound, null, "Property '{0}' is of type Custom but has no custom members.", inputProperty.Name));
}
return PropertyValidationOutcome.Ok;
}
else
{
var odataSchema = schemas[schemaPropertyDef.ODataTypeName];
if (inputProperty.CustomMembers == null)
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.MissingCustomMembers, null, "Property {0} is missing custom members and cannot be validated.", inputProperty.Name));
return PropertyValidationOutcome.InvalidType;
}
else
{
odataSchema.ValidateObjectProperties(inputProperty.CustomMembers.Values, options, schemas, detectedErrors);
return PropertyValidationOutcome.Ok;
}
}
}
else if (schemaPropertyDef.Type == JsonDataType.Object)
{
detectedErrors.Add(new ValidationWarning(ValidationErrorCode.CustomValidationNotSupported, null, "Schema type was 'Custom' which is not supported. Add a resource type to the definition of property: {0}", inputProperty.Name));
return PropertyValidationOutcome.MissingResourceType;
}
else
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.ExpectedTypeDifferent, null, "Type mismatch: property '{0}' [{1}] doesn't match expected type [{2}].",
inputProperty.Name, inputProperty.Type, schemaPropertyDef.Type));
return PropertyValidationOutcome.InvalidType;
}
}
else
{
detectedErrors.Add(new ValidationWarning(ValidationErrorCode.AdditionalPropertyDetected, null, "Extra property: property '{0}' [{1}] was not expected.", inputProperty.Name, inputProperty.Type));
return PropertyValidationOutcome.MissingFromSchema;
}
}
/// <summary>
/// Validates that the actual response body matches the schema defined for the response and any additional constraints
/// from the expected request (e.g. properties that are included in the expected response are required in the actual
/// response even if the metadata defines that the response is truncated)
/// </summary>
/// <param name="method"></param>
/// <param name="actualResponse"></param>
/// <param name="expectedResponse"></param>
/// <param name="schemaErrors"></param>
/// <returns></returns>
internal bool ValidateResponseMatchesSchema(MethodDefinition method, HttpResponse actualResponse, HttpResponse expectedResponse, out ValidationError[] schemaErrors, ValidationOptions options = null)
{
List <ValidationError> newErrors = new List <ValidationError>();
var expectedResourceType = method.ExpectedResponseMetadata.ResourceType;
switch (expectedResourceType)
{
case "stream":
case "Stream":
// No validation since we're streaming data
schemaErrors = new ValidationError[0];
return(true);
case "string":
case "String":
case "Edm.String":
case " Edm.string":
schemaErrors = new ValidationError[0];
return(true);
}
// Get a reference of our JsonSchema that we're checking the response with
var expectedResponseJson = (null != expectedResponse) ? expectedResponse.Body : null;
JsonSchema schema = this.GetJsonSchema(expectedResourceType, newErrors, expectedResponseJson);
if (null == schema)
{
newErrors.Add(new ValidationError(ValidationErrorCode.ResourceTypeNotFound, null, "Unable to locate a definition for resource type: {0}", expectedResourceType));
}
else
{
ValidationError[] validationJsonOutput;
this.ValidateJsonCompilesWithSchema(schema, new JsonExample(actualResponse.Body, method.ExpectedResponseMetadata), out validationJsonOutput, (null != expectedResponseJson) ? new JsonExample(expectedResponseJson) : null, options);
newErrors.AddRange(validationJsonOutput);
}
schemaErrors = newErrors.ToArray();
return(!schemaErrors.WereWarningsOrErrors());
}
/// <summary>
/// Check each member of the actualProperty's array to make sure it matches the resource type specified for the property.
/// </summary>
/// <param name="actualProperty"></param>
/// <param name="schemas"></param>
/// <param name="detectedErrors"></param>
/// <param name="options"></param>
private PropertyValidationOutcome ValidateArrayProperty(JsonProperty actualProperty, Dictionary<string, JsonSchema> schemas, List<ValidationError> detectedErrors, ValidationOptions options)
{
JArray actualArray = (JArray)JsonConvert.DeserializeObject(actualProperty.OriginalValue);
JsonSchema memberSchema;
if (string.IsNullOrEmpty(actualProperty.ODataTypeName))
{
return this.ValidateSimpleArrayProperty(actualProperty, this.ExpectedProperties[actualProperty.Name], detectedErrors);
}
else if (!schemas.TryGetValue(actualProperty.ODataTypeName, out memberSchema))
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.ResourceTypeNotFound, null, "Failed to locate resource definition for: {0}", actualProperty.ODataTypeName));
return PropertyValidationOutcome.MissingResourceType;
}
bool hadErrors = false;
for(int i=0; i<actualArray.Count; i++)
{
JContainer member = actualArray[i] as JContainer;
if (member != null)
{
List<ValidationError> memberErrors = new List<ValidationError>();
memberSchema.ValidateContainerObject(member, options, schemas, memberErrors);
hadErrors |= memberErrors.Count > 0;
foreach (var error in memberErrors)
{
error.Source = string.Format("{0} [{1}]", actualProperty.Name, i);
detectedErrors.Add(error);
}
}
}
return hadErrors ? PropertyValidationOutcome.GenericError : PropertyValidationOutcome.Ok;
}
private bool ValidateCustomObject(JsonProperty[] properties, out ValidationError[] errors, Dictionary<string, JsonSchema> otherSchemas, ValidationOptions options)
{
List<string> missingProperties = new List<string>(this.ExpectedProperties.Keys);
List<ValidationError> detectedErrors = new List<ValidationError>();
foreach (var inputProperty in properties)
{
missingProperties.Remove(inputProperty.Name);
this.ValidateProperty(inputProperty, otherSchemas, detectedErrors, new ValidationOptions());
}
this.CleanMissingProperties(options, missingProperties);
errors = detectedErrors.ToArray();
return detectedErrors.Count == 0;
}
/// <summary>
/// Check each member of the actualProperty's array to make sure it matches the resource type specified for the property.
/// </summary>
/// <param name="actualProperty"></param>
/// <param name="schemas"></param>
/// <param name="detectedErrors"></param>
/// <param name="options"></param>
private PropertyValidationOutcome ValidateArrayProperty(ParameterDefinition actualProperty, Dictionary <string, JsonSchema> schemas, List <ValidationError> detectedErrors, ValidationOptions options)
{
JArray actualArray = (JArray)JsonConvert.DeserializeObject(actualProperty.OriginalValue);
JsonSchema memberSchema;
if (string.IsNullOrEmpty(actualProperty.Type.CustomTypeName))
{
return(this.ValidateSimpleArrayProperty(actualProperty, this.ExpectedProperties[actualProperty.Name], detectedErrors));
}
else if (!schemas.TryGetValue(actualProperty.Type.CustomTypeName, out memberSchema))
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.ResourceTypeNotFound, null, "Failed to locate resource definition for: {0}", actualProperty.Type.CustomTypeName));
return(PropertyValidationOutcome.MissingResourceType);
}
bool hadErrors = false;
for (int i = 0; i < actualArray.Count; i++)
{
JContainer member = actualArray[i] as JContainer;
if (member != null)
{
List <ValidationError> memberErrors = new List <ValidationError>();
memberSchema.ValidateContainerObject(member, options, schemas, memberErrors);
// TODO: Filter out non-unique errors
hadErrors |= memberErrors.Count > 0;
detectedErrors.AddUniqueErrors(memberErrors);
}
}
return(hadErrors ? PropertyValidationOutcome.GenericError : PropertyValidationOutcome.Ok);
}
/// <summary>
/// Validate the input json against the defined scehma when the instance was created.
/// </summary>
/// <param name="jsonInput">Input json to validate against schema</param>
/// <param name="errors">Array of errors if the validation fails</param>
/// <param name="otherSchemas"></param>
/// <param name="options"></param>
/// <param name="expectedJson"></param>
/// <returns>True if validation was successful, otherwise false.</returns>
public bool ValidateJson(JsonExample jsonInput, out ValidationError[] errors, Dictionary<string, JsonSchema> otherSchemas, ValidationOptions options, JsonExample expectedJson = null)
{
JContainer obj;
try
{
var settings = new JsonSerializerSettings { DateParseHandling = DateParseHandling.None, NullValueHandling = NullValueHandling.Include, DefaultValueHandling = DefaultValueHandling.Include };
obj = (JContainer)JsonConvert.DeserializeObject(jsonInput.JsonData, settings);
}
catch (Exception ex)
{
errors = new ValidationError[] { new ValidationError(ValidationErrorCode.JsonParserException, null, "Failed to parse json string: {0}. Json: {1}", ex.Message, jsonInput.JsonData) };
return false;
}
var annotation = jsonInput.Annotation ?? new CodeBlockAnnotation();
List<ValidationError> detectedErrors = new List<ValidationError>();
bool expectErrorObject = (jsonInput.Annotation != null) && jsonInput.Annotation.ExpectError;
// Check for an error response
dynamic errorObject = obj["error"];
if (null != errorObject && !expectErrorObject)
{
string code = errorObject.code;
string message = errorObject.message;
detectedErrors.Clear();
detectedErrors.Add(new ValidationError(ValidationErrorCode.JsonErrorObject, null, "Error response received. Code: {0}, Message: {1}", code, message));
errors = detectedErrors.ToArray();
return false;
}
else if (expectErrorObject && null == errorObject)
{
detectedErrors.Clear();
detectedErrors.Add(new ValidationError(ValidationErrorCode.JsonErrorObjectExpected, null, "Expected an error object response, but didn't receive one."));
errors = detectedErrors.ToArray();
return false;
}
// Check to see if this is a "collection" instance
if (null != annotation && annotation.IsCollection)
{
this.ValidateCollectionObject(obj, annotation, otherSchemas, options.CollectionPropertyName, detectedErrors);
}
// otherwise verify the object matches this schema
else
{
options = options ?? new ValidationOptions(annotation);
if (null != expectedJson)
{
var expectedJsonSchema = new JsonSchema(expectedJson.JsonData, expectedJson.Annotation);
options.ExpectedJsonSchema = expectedJsonSchema;
options.RequiredPropertyNames = expectedJsonSchema.ExpectedProperties.Keys.ToArray();
}
this.ValidateContainerObject(obj, options, otherSchemas, detectedErrors);
}
errors = detectedErrors.ToArray();
return detectedErrors.Count == 0;
}
/// <summary>
/// Creates a new ValidationOptions instance inheriting values from this instance, and modified for the specific options of a property.
/// </summary>
/// <param name="propertyName"></param>
/// <returns></returns>
internal ValidationOptions CreateForProperty(string propertyName)
{
var newOption = new ValidationOptions
{
AllowTruncatedResponses = this.AllowTruncatedResponses,
CollectionPropertyName = this.CollectionPropertyName,
RelaxedStringValidation = this.RelaxedStringValidation
};
if (null != this.ExpectedJsonSchema)
{
var propertyData = this.ExpectedJsonSchema.Properties.FirstOrDefault(x => x.Name.Equals(propertyName));
if (null != propertyData && null != propertyData.OriginalValue)
{
// Compute a new expected schema for the property's contents.
newOption.ExpectedJsonSchema = new JsonSchema(propertyData.OriginalValue, new CodeBlockAnnotation());
newOption.RequiredPropertyNames = newOption.ExpectedJsonSchema.Properties.Select(x => x.Name).ToArray();
}
else
{
newOption.ExpectedJsonSchema = null;
}
}
else
{
newOption.RequiredPropertyNames = this.RequiredPropertyNames;
}
return newOption;
}
private void ValidateObjectProperties(IEnumerable <ParameterDefinition> propertiesOnObject, ValidationOptions options, Dictionary <string, JsonSchema> otherSchemas, List <ValidationError> detectedErrors)
{
List <string> missingProperties = new List <string>();
missingProperties.AddRange(from m in this.ExpectedProperties select m.Key);
foreach (var property in propertiesOnObject)
{
missingProperties.Remove(property.Name);
// This detects bad types, extra properties, etc.
if (null != options && (property.Type.IsCollection || property.Type.IsObject))
{
var propertyOptions = options.CreateForProperty(property.Name);
this.ValidateProperty(property, otherSchemas, detectedErrors, propertyOptions);
}
else
{
this.ValidateProperty(property, otherSchemas, detectedErrors, options);
}
}
this.CleanMissingProperties(options, missingProperties);
if (missingProperties.Count > 0)
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.RequiredPropertiesMissing, null, "Missing properties: response was missing these required properties: {0}", missingProperties.ComponentsJoinedByString(", ")));
}
}
/// <summary>
/// Verify that a property from the json-to-validate matches something in our schema
/// </summary>
/// <param name="inputProperty"></param>
/// <param name="schemas"></param>
/// <param name="detectedErrors"></param>
/// <param name="options"></param>
/// <returns></returns>
private PropertyValidationOutcome ValidateProperty(ParameterDefinition inputProperty, Dictionary<string, JsonSchema> schemas, List<ValidationError> detectedErrors, ValidationOptions options)
{
if (this.ExpectedProperties.ContainsKey(inputProperty.Name))
{
// The property was expected to be found in this schema! Yay.
var schemaPropertyDef = this.ExpectedProperties[inputProperty.Name];
// Check for simple value types first
if (this.SimpleValueTypes(schemaPropertyDef.Type, inputProperty.Type) && this.AllFalse(schemaPropertyDef.Type.IsCollection, inputProperty.Type.IsCollection))
{
return ValidateSameDataType(schemaPropertyDef, inputProperty, detectedErrors, (null != options) ? options.RelaxedStringValidation : false);
}
else if (null == inputProperty.OriginalValue)
{
if (null != this.NullableProperties && !this.NullableProperties.Contains(schemaPropertyDef.Name))
{
detectedErrors.Add(new ValidationWarning(ValidationErrorCode.NullPropertyValue, null, "Non-nullable property {0} had a null value in the response. Expected {1}.", schemaPropertyDef.Name, schemaPropertyDef.Type));
}
return PropertyValidationOutcome.Ok;
}
else if (schemaPropertyDef.Type.IsCollection || inputProperty.Type.IsCollection)
{
// Check for an array
if (schemaPropertyDef.Type.IsCollection && !inputProperty.Type.IsCollection)
{
// Expected an array, but didn't get one
detectedErrors.Add(new ValidationError(ValidationErrorCode.ExpectedArrayValue, null, "Expected an array but property was not an array: {0}", inputProperty.Name));
return PropertyValidationOutcome.InvalidType;
}
else if (!schemaPropertyDef.Type.IsCollection && inputProperty.Type.IsCollection)
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.ExpectedNonArrayValue, null, "Expected a value of type {0} but property was an array: {1}", schemaPropertyDef.Type, inputProperty.Name));
return PropertyValidationOutcome.InvalidType;
}
return this.ValidateArrayProperty(inputProperty, schemas, detectedErrors, options);
}
else if (schemaPropertyDef.Type.IsObject && inputProperty.Type.IsObject)
{
// Compare the ODataType schema to the custom schema
if (null == schemaPropertyDef.Type.CustomTypeName || !schemas.ContainsKey(schemaPropertyDef.Type.CustomTypeName))
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.ResourceTypeNotFound, null, "Missing resource: resource {0} was not found (property name '{1}').", schemaPropertyDef.Type.CustomTypeName, inputProperty.Name));
return PropertyValidationOutcome.MissingResourceType;
}
else if (inputProperty.Type.IsObject)
{
var odataSchema = schemas[schemaPropertyDef.Type.CustomTypeName];
ValidationError[] odataErrors;
if (null != inputProperty.Type.CustomMembers && !odataSchema.ValidateCustomObject(inputProperty.Type.CustomMembers.ToArray(), out odataErrors, schemas, options))
{
var propertyError = ValidationError.NewConsolidatedError(ValidationErrorCode.ConsolidatedError, odataErrors, "Schemas validation failed on property '{0}' ['{1}']", inputProperty.Name, odataSchema.ResourceName);
detectedErrors.Add(propertyError);
return PropertyValidationOutcome.InvalidType;
}
else if (null == inputProperty.Type.CustomMembers)
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.NoCustomMembersFound, null, "Property '{0}' is of type Custom but has no custom members.", inputProperty.Name));
}
return PropertyValidationOutcome.Ok;
}
else
{
var odataSchema = schemas[schemaPropertyDef.Type.CustomTypeName];
if (inputProperty.Type.CustomMembers == null)
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.MissingCustomMembers, null, "Property {0} is missing custom members and cannot be validated.", inputProperty.Name));
return PropertyValidationOutcome.InvalidType;
}
else
{
odataSchema.ValidateObjectProperties(inputProperty.Type.CustomMembers, options, schemas, detectedErrors);
return PropertyValidationOutcome.Ok;
}
}
}
else if (schemaPropertyDef.Type.IsObject)
{
detectedErrors.Add(new ValidationWarning(ValidationErrorCode.CustomValidationNotSupported, null, "Schemas type was 'Custom' which is not supported. Add a resource type to the definition of property: {0}", inputProperty.Name));
return PropertyValidationOutcome.MissingResourceType;
}
else
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.ExpectedTypeDifferent, null, "Type mismatch: property '{0}' [{1}] doesn't match expected type [{2}].",
inputProperty.Name, inputProperty.Type, schemaPropertyDef.Type));
return PropertyValidationOutcome.InvalidType;
}
}
else
{
// Check to see if this property is on the ignorable list
string[] ignorableUndocumentedProperties = this.OriginalResource?.SourceFile.Parent.Requirements?.IgnorableProperties;
string propertyName = inputProperty.Name;
string annotationName = null;
var indexOfAtSign = propertyName.IndexOf('@');
if (indexOfAtSign > 0)
{
// [email protected] is an example of what we're looking for here
annotationName = propertyName.Substring(indexOfAtSign);
propertyName = propertyName.Substring(0, indexOfAtSign);
}
if (null != annotationName)
{
// Check to see if propertyName is known or not. If it isn't known, fail.
if (this.Properties.Any(x => x.Name.Equals(propertyName)))
{
// If the cleaned up propertyName is known, then check to see if the annotation is ignorable
if (null != ignorableUndocumentedProperties && ignorableUndocumentedProperties.Contains(annotationName))
{
// If we know of both the property and the annotation, we're good.
return PropertyValidationOutcome.Ok;
}
}
}
if (null != ignorableUndocumentedProperties && ignorableUndocumentedProperties.Contains(propertyName))
{
return PropertyValidationOutcome.Ok;
}
// This property isn't legit
detectedErrors.Add(new ValidationWarning(ValidationErrorCode.AdditionalPropertyDetected, null, "Undocumented property '{0}' [{1}] was not expected on resource {2}.", inputProperty.Name, inputProperty.Type, this.ResourceName));
return PropertyValidationOutcome.MissingFromSchema;
}
}
private void ValidateCollectionObject(JContainer obj, CodeBlockAnnotation annotation, Dictionary<string, JsonSchema> otherSchemas, string collectionPropertyName, List<ValidationError> detectedErrors, ValidationOptions options)
{
// TODO: also validate additional properties on the collection, like nextDataLink
var collection = obj[collectionPropertyName];
if (null == collection)
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.MissingCollectionProperty, null, "Failed to locate collection property '{0}' in response.", collectionPropertyName));
}
else
{
var collectionMembers = obj[collectionPropertyName];
if (!collectionMembers.Any())
{
if (!annotation.IsEmpty)
{
detectedErrors.Add(
new ValidationWarning(
ValidationErrorCode.CollectionArrayEmpty,
null,
"Property contained an empty array that was not validated: {0}",
collectionPropertyName));
}
}
else if (annotation.IsEmpty)
{
detectedErrors.Add(
new ValidationWarning(
ValidationErrorCode.CollectionArrayNotEmpty,
null,
"Property contained a non-empty array that was expected to be empty: {0}",
collectionPropertyName));
}
foreach (var jToken in collectionMembers)
{
var container = jToken as JContainer;
if (null != container)
{
List<ValidationError> containerErrors = new List<ValidationError>();
var deeperOptions = new ValidationOptions(options)
{
AllowTruncatedResponses = annotation.TruncatedResult
};
this.ValidateContainerObject(
container,
deeperOptions,
otherSchemas,
containerErrors);
detectedErrors.AddUniqueErrors(containerErrors);
}
}
}
}
/// <summary>
/// Check each member of the actualProperty's array to make sure it matches the resource type specified for the property.
/// </summary>
/// <param name="actualProperty"></param>
/// <param name="schemas"></param>
/// <param name="detectedErrors"></param>
/// <param name="options"></param>
private PropertyValidationOutcome ValidateArrayProperty(ParameterDefinition actualProperty, Dictionary<string, JsonSchema> schemas, List<ValidationError> detectedErrors, ValidationOptions options)
{
JArray actualArray = null;
try
{
actualArray = (JArray)JsonConvert.DeserializeObject(actualProperty.OriginalValue);
}
catch (InvalidCastException ex)
{
throw new InvalidCastException($"Property {actualProperty.Name} expected to be an array, but failed to cast value to an array: {actualProperty.OriginalValue}");
}
var expectedPropertyDefinition = this.ExpectedProperties[actualProperty.Name];
JsonSchema memberSchema = null;
if (actualProperty.Type.CollectionResourceType == SimpleDataType.Object && expectedPropertyDefinition.Type.CustomTypeName != null)
{
// We have an ambigious array, but we know what it's supposed to be so let's use that
schemas.TryGetValue(expectedPropertyDefinition.Type.CustomTypeName, out memberSchema);
}
if (memberSchema == null && string.IsNullOrEmpty(actualProperty.Type.CustomTypeName))
{
return this.ValidateSimpleArrayProperty(actualProperty, this.ExpectedProperties[actualProperty.Name], detectedErrors);
}
else if (memberSchema == null && !schemas.TryGetValue(actualProperty.Type.CustomTypeName, out memberSchema))
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.ResourceTypeNotFound, null, "Failed to locate resource definition for: {0}", actualProperty.Type.CustomTypeName));
return PropertyValidationOutcome.MissingResourceType;
}
bool hadErrors = false;
for(int i=0; i<actualArray.Count; i++)
{
JContainer member = actualArray[i] as JContainer;
if (member != null)
{
List<ValidationError> memberErrors = new List<ValidationError>();
memberSchema.ValidateContainerObject(member, options, schemas, memberErrors);
// TODO: Filter out non-unique errors
hadErrors |= memberErrors.Count > 0;
detectedErrors.AddUniqueErrors(memberErrors);
}
}
return hadErrors ? PropertyValidationOutcome.GenericError : PropertyValidationOutcome.Ok;
}
/// <summary>
/// Validates the value of json according to an implicit schmea defined by expectedJson
/// </summary>
/// <param name="expectedResponseAnnotation"></param>
/// <param name="actualResponseBodyJson"></param>
/// <param name="errors"></param>
/// <returns></returns>
public bool ValidateJsonExample(CodeBlockAnnotation expectedResponseAnnotation, string actualResponseBodyJson, out ValidationError[] errors, ValidationOptions options = null)
{
List <ValidationError> newErrors = new List <ValidationError>();
var resourceType = expectedResponseAnnotation.ResourceType;
if (resourceType == "stream")
{
// No validation since we're streaming data
errors = null;
return(true);
}
else
{
JsonSchema schema;
if (string.IsNullOrEmpty(resourceType))
{
schema = JsonSchema.EmptyResponseSchema;
}
else if (!this.registeredSchema.TryGetValue(resourceType, out schema))
{
newErrors.Add(new ValidationWarning(ValidationErrorCode.ResponseResourceTypeMissing, null, "Missing required resource: {0}. Validation limited to basics only.", resourceType));
// Create a new schema based on what's avaiable in the json
schema = new JsonSchema(actualResponseBodyJson, new CodeBlockAnnotation {
ResourceType = expectedResponseAnnotation.ResourceType
});
}
ValidationError[] validationJsonOutput;
this.ValidateJsonCompilesWithSchema(schema, new JsonExample(actualResponseBodyJson, expectedResponseAnnotation), out validationJsonOutput, options: options);
newErrors.AddRange(validationJsonOutput);
errors = newErrors.ToArray();
return(errors.Length == 0);
}
}
/// <summary>
/// Examines input json string to ensure that it compiles with the JsonSchema definition. Any errors in the
/// validation of the schema are returned via the errors out parameter.
/// </summary>
/// <param name="schema">Schemas definition used as a reference.</param>
/// <param name="inputJson">Input json example to be validated</param>
/// <param name="errors">Out parameter that provides any errors, warnings, or messages that were generated</param>
/// <param name="expectedJson"></param>
/// <returns></returns>
public bool ValidateJsonCompilesWithSchema(JsonSchema schema, JsonExample inputJson, out ValidationError[] errors, JsonExample expectedJson = null, ValidationOptions options = null)
{
if (null == schema)
{
throw new ArgumentNullException("schema");
}
if (null == inputJson)
{
throw new ArgumentNullException("inputJson");
}
string collectionPropertyName = "value";
if (null != inputJson.Annotation && null != inputJson.Annotation.CollectionPropertyName)
{
collectionPropertyName = inputJson.Annotation.CollectionPropertyName;
}
// If we didn't get an options, create a new one with some defaults provided by the annotation
options = options ?? new ValidationOptions();
options.AllowTruncatedResponses = (inputJson.Annotation ?? new CodeBlockAnnotation()).TruncatedResult;
options.CollectionPropertyName = collectionPropertyName;
return(schema.ValidateJson(inputJson, out errors, this.registeredSchema, options, expectedJson));
}
/// <summary>
/// Verify that a property from the json-to-validate matches something in our schema
/// </summary>
/// <param name="inputProperty"></param>
/// <param name="schemas"></param>
/// <param name="detectedErrors"></param>
/// <param name="options"></param>
/// <returns></returns>
private PropertyValidationOutcome ValidateProperty(ParameterDefinition inputProperty, Dictionary <string, JsonSchema> schemas, List <ValidationError> detectedErrors, ValidationOptions options)
{
if (this.ExpectedProperties.ContainsKey(inputProperty.Name))
{
// The property was expected to be found in this schema! Yay.
var schemaPropertyDef = this.ExpectedProperties[inputProperty.Name];
// Check for simple value types first
if (this.SimpleValueTypes(schemaPropertyDef.Type, inputProperty.Type) && this.AllFalse(schemaPropertyDef.Type.IsCollection, inputProperty.Type.IsCollection))
{
return(ValidateSameDataType(schemaPropertyDef, inputProperty, detectedErrors, (null != options) ? options.RelaxedStringValidation : false));
}
else if (null == inputProperty.OriginalValue)
{
if (null != this.NullableProperties && !this.NullableProperties.Contains(schemaPropertyDef.Name))
{
detectedErrors.Add(new ValidationWarning(ValidationErrorCode.NullPropertyValue, null, "Non-nullable property {0} had a null value in the response. Expected {1}.", schemaPropertyDef.Name, schemaPropertyDef.Type));
}
return(PropertyValidationOutcome.Ok);
}
else if (schemaPropertyDef.Type.IsCollection || inputProperty.Type.IsCollection)
{
// Check for an array
if (schemaPropertyDef.Type.IsCollection && !inputProperty.Type.IsCollection)
{
// Expected an array, but didn't get one
detectedErrors.Add(new ValidationError(ValidationErrorCode.ExpectedArrayValue, null, "Expected an array but property was not an array: {0}", inputProperty.Name));
return(PropertyValidationOutcome.InvalidType);
}
else if (!schemaPropertyDef.Type.IsCollection && inputProperty.Type.IsCollection)
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.ExpectedNonArrayValue, null, "Expected a value of type {0} but property was an array: {1}", schemaPropertyDef.Type, inputProperty.Name));
return(PropertyValidationOutcome.InvalidType);
}
return(this.ValidateArrayProperty(inputProperty, schemas, detectedErrors, options));
}
else if (schemaPropertyDef.Type.IsObject && inputProperty.Type.IsObject)
{
// Compare the ODataType schema to the custom schema
if (null == schemaPropertyDef.Type.CustomTypeName || !schemas.ContainsKey(schemaPropertyDef.Type.CustomTypeName))
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.ResourceTypeNotFound, null, "Missing resource: resource {0} was not found (property name '{1}').", schemaPropertyDef.Type.CustomTypeName, inputProperty.Name));
return(PropertyValidationOutcome.MissingResourceType);
}
else if (inputProperty.Type.IsObject)
{
var odataSchema = schemas[schemaPropertyDef.Type.CustomTypeName];
ValidationError[] odataErrors;
if (null != inputProperty.Type.CustomMembers && !odataSchema.ValidateCustomObject(inputProperty.Type.CustomMembers.ToArray(), out odataErrors, schemas, options))
{
var propertyError = ValidationError.NewConsolidatedError(ValidationErrorCode.ConsolidatedError, odataErrors, "Schemas validation failed on property '{0}' ['{1}']", inputProperty.Name, odataSchema.ResourceName);
detectedErrors.Add(propertyError);
return(PropertyValidationOutcome.InvalidType);
}
else if (null == inputProperty.Type.CustomMembers)
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.NoCustomMembersFound, null, "Property '{0}' is of type Custom but has no custom members.", inputProperty.Name));
}
return(PropertyValidationOutcome.Ok);
}
else
{
var odataSchema = schemas[schemaPropertyDef.Type.CustomTypeName];
if (inputProperty.Type.CustomMembers == null)
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.MissingCustomMembers, null, "Property {0} is missing custom members and cannot be validated.", inputProperty.Name));
return(PropertyValidationOutcome.InvalidType);
}
else
{
odataSchema.ValidateObjectProperties(inputProperty.Type.CustomMembers, options, schemas, detectedErrors);
return(PropertyValidationOutcome.Ok);
}
}
}
else if (schemaPropertyDef.Type.IsObject)
{
detectedErrors.Add(new ValidationWarning(ValidationErrorCode.CustomValidationNotSupported, null, "Schemas type was 'Custom' which is not supported. Add a resource type to the definition of property: {0}", inputProperty.Name));
return(PropertyValidationOutcome.MissingResourceType);
}
else
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.ExpectedTypeDifferent, null, "Type mismatch: property '{0}' [{1}] doesn't match expected type [{2}].",
inputProperty.Name, inputProperty.Type, schemaPropertyDef.Type));
return(PropertyValidationOutcome.InvalidType);
}
}
else
{
detectedErrors.Add(new ValidationWarning(ValidationErrorCode.AdditionalPropertyDetected, null, "Undocumented property '{0}' [{1}] was not expected.", inputProperty.Name, inputProperty.Type));
return(PropertyValidationOutcome.MissingFromSchema);
}
}
/// <summary>
/// Validates that the actual response body matches the schema defined for the response and any additional constraints
/// from the expected request (e.g. properties that are included in the expected response are required in the actual
/// response even if the metadata defines that the response is truncated)
/// </summary>
/// <param name="method"></param>
/// <param name="actualResponse"></param>
/// <param name="expectedResponse"></param>
/// <param name="schemaErrors"></param>
/// <returns></returns>
internal bool ValidateResponseMatchesSchema(MethodDefinition method, HttpResponse actualResponse, HttpResponse expectedResponse, out ValidationError[] schemaErrors, ValidationOptions options = null)
{
List<ValidationError> newErrors = new List<ValidationError>();
var expectedResourceType = method.ExpectedResponseMetadata.ResourceType;
switch (expectedResourceType)
{
case "stream":
case "Stream":
// No validation since we're streaming data
schemaErrors = new ValidationError[0];
return true;
case "string":
case "String":
case "Edm.String":
case" Edm.string":
schemaErrors = new ValidationError[0];
return true;
}
// Get a reference of our JsonSchema that we're checking the response with
var expectedResponseJson = (null != expectedResponse) ? expectedResponse.Body : null;
JsonSchema schema = this.GetJsonSchema(expectedResourceType, newErrors, expectedResponseJson);
if (null == schema)
{
newErrors.Add(new ValidationError(ValidationErrorCode.ResourceTypeNotFound, null, "Unable to locate a definition for resource type: {0}", expectedResourceType));
}
else
{
ValidationError[] validationJsonOutput;
this.ValidateJsonCompilesWithSchema(schema, new JsonExample(actualResponse.Body, method.ExpectedResponseMetadata), out validationJsonOutput, (null != expectedResponseJson) ? new JsonExample(expectedResponseJson) : null, options);
newErrors.AddRange(validationJsonOutput);
}
schemaErrors = newErrors.ToArray();
return !schemaErrors.WereWarningsOrErrors();
}
private bool ValidateCustomObject(ParameterDefinition[] properties, out ValidationError[] errors, Dictionary <string, JsonSchema> otherSchemas, ValidationOptions options)
{
List <string> missingProperties = new List <string>(this.ExpectedProperties.Keys);
List <ValidationError> detectedErrors = new List <ValidationError>();
foreach (var inputProperty in properties)
{
missingProperties.Remove(inputProperty.Name);
this.ValidateProperty(inputProperty, otherSchemas, detectedErrors, options);
}
this.CleanMissingProperties(options, missingProperties);
errors = detectedErrors.ToArray();
return(detectedErrors.Count == 0);
}
/// <summary>
/// Validates that a particular HttpResponse matches the method definition and optionally the expected response.
/// </summary>
/// <param name="method">Method definition that was used to generate a request.</param>
/// <param name="actualResponse">Actual response from the service (this is what we validate).</param>
/// <param name="expectedResponse">Prototype response (expected) that shows what a valid response should look like.</param>
/// <param name="scenario">A test scenario used to generate the response, which may include additional parameters to verify.</param>
/// <param name="errors">A collection of errors, warnings, and verbose messages generated by this process.</param>
public void ValidateResponse(HttpResponse actualResponse, HttpResponse expectedResponse, ScenarioDefinition scenario, out ValidationError[] errors, ValidationOptions options = null)
{
if (null == actualResponse) throw new ArgumentNullException("actualResponse");
List<ValidationError> detectedErrors = new List<ValidationError>();
// Verify the request is valid (headers, etc)
this.VerifyHttpRequest(detectedErrors);
// Verify that the expected response headers match the actual response headers
ValidationError[] httpErrors;
if (null != expectedResponse && !expectedResponse.ValidateResponseHeaders(actualResponse, out httpErrors, (null != scenario) ? scenario.AllowedStatusCodes : null))
{
detectedErrors.AddRange(httpErrors);
}
// Verify the actual response body is correct according to the schema defined for the response
ValidationError[] bodyErrors;
this.VerifyResponseBody(actualResponse, expectedResponse, out bodyErrors, options);
detectedErrors.AddRange(bodyErrors);
// Verify any expectations in the scenario are met
if (null != scenario)
{
scenario.ValidateExpectations(actualResponse, detectedErrors);
}
errors = detectedErrors.ToArray();
}
/// <summary>
/// Modifies the missingProperties list to remove optional properties
/// and handle truncated or required properties.
/// </summary>
/// <param name="options"></param>
/// <param name="missingProperties"></param>
private void CleanMissingProperties(ValidationOptions options, List<string> missingProperties)
{
if (null == options)
return;
// Ignore any missing properties that are defined in the schema as optional
missingProperties.RemoveRange(this.OptionalProperties);
if (ValidationConfig.ExpectedResponseAsRequiredProperties &&
options.AllowTruncatedResponses && null != options.RequiredPropertyNames)
{
// Ignore any missing properties that aren't in the required properties list
missingProperties.IntersectInPlace(options.RequiredPropertyNames);
}
else if (options.AllowTruncatedResponses)
{
// Ignore all missing properties
missingProperties.Clear();
}
}
/// <summary>
/// Verify that the body of the actual response is consistent with the method definition and expected response parameters
/// </summary>
/// <param name="method">The MethodDefinition that generated the response.</param>
/// <param name="actualResponse">The actual response from the service to validate.</param>
/// <param name="expectedResponse">The prototype expected response from the service.</param>
/// <param name="detectedErrors">A collection of errors that will be appended with any detected errors</param>
private void VerifyResponseBody(HttpResponse actualResponse, HttpResponse expectedResponse, out ValidationError[] errors, ValidationOptions options = null)
{
List<ValidationError> detectedErrors = new List<ValidationError>();
if (string.IsNullOrEmpty(actualResponse.Body) &&
(expectedResponse != null && !string.IsNullOrEmpty(expectedResponse.Body)))
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.HttpBodyExpected, null, "Body missing from response (expected response includes a body or a response type was provided)."));
}
else if (!string.IsNullOrEmpty(actualResponse.Body))
{
ValidationError[] schemaErrors;
if (this.ExpectedResponseMetadata == null ||
(string.IsNullOrEmpty(this.ExpectedResponseMetadata.ResourceType) &&
(expectedResponse != null && !string.IsNullOrEmpty(expectedResponse.Body))))
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.ResponseResourceTypeMissing, null, "Expected a response, but resource type on method is missing: {0}", this.Identifier));
}
else
{
var otherResources = this.SourceFile.Parent.ResourceCollection;
if (
!otherResources.ValidateResponseMatchesSchema(
this,
actualResponse,
expectedResponse,
out schemaErrors,
options))
{
detectedErrors.AddRange(schemaErrors);
}
}
var responseValidation = actualResponse.IsResponseValid(
this.SourceFile.DisplayName,
this.SourceFile.Parent.Requirements);
detectedErrors.AddRange(responseValidation.Messages);
}
errors = detectedErrors.ToArray();
}
/// <summary>
/// Verify that a Json container (object) is valid according to it's resource name (schema).
/// </summary>
/// <param name="obj"></param>
/// <param name="options"></param>
/// <param name="otherSchemas"></param>
/// <param name="detectedErrors"></param>
private void ValidateContainerObject(JContainer obj, ValidationOptions options, Dictionary<string, JsonSchema> otherSchemas, List<ValidationError> detectedErrors)
{
var containerProperties = from p in obj
select ParseProperty(p, this, detectedErrors);
this.ValidateObjectProperties(containerProperties.Where(x => null != x), options, otherSchemas, detectedErrors);
}
/// <summary>
/// Verify that a property from the json-to-validate matches something in our schema
/// </summary>
/// <param name="inputProperty"></param>
/// <param name="schemas"></param>
/// <param name="detectedErrors"></param>
/// <param name="options"></param>
/// <returns></returns>
private PropertyValidationOutcome ValidateProperty(ParameterDefinition inputProperty, Dictionary <string, JsonSchema> schemas, List <ValidationError> detectedErrors, ValidationOptions options)
{
if (this.ExpectedProperties.ContainsKey(inputProperty.Name))
{
// The property was expected to be found in this schema! Yay.
var schemaPropertyDef = this.ExpectedProperties[inputProperty.Name];
// Check for simple value types first
if (this.SimpleValueTypes(schemaPropertyDef.Type, inputProperty.Type) && this.AllFalse(schemaPropertyDef.Type.IsCollection, inputProperty.Type.IsCollection))
{
return(ValidateSameDataType(schemaPropertyDef, inputProperty, detectedErrors, (null != options) ? options.RelaxedStringValidation : false));
}
else if (null == inputProperty.OriginalValue)
{
if (null != this.NullableProperties && !this.NullableProperties.Contains(schemaPropertyDef.Name))
{
detectedErrors.Add(new ValidationWarning(ValidationErrorCode.NullPropertyValue, null, "Non-nullable property {0} had a null value in the response. Expected {1}.", schemaPropertyDef.Name, schemaPropertyDef.Type));
}
return(PropertyValidationOutcome.Ok);
}
else if (schemaPropertyDef.Type.IsCollection || inputProperty.Type.IsCollection)
{
// Check for an array
if (schemaPropertyDef.Type.IsCollection && !inputProperty.Type.IsCollection)
{
// Expected an array, but didn't get one
detectedErrors.Add(new ValidationError(ValidationErrorCode.ExpectedArrayValue, null, "Expected an array but property was not an array: {0}", inputProperty.Name));
return(PropertyValidationOutcome.InvalidType);
}
else if (!schemaPropertyDef.Type.IsCollection && inputProperty.Type.IsCollection)
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.ExpectedNonArrayValue, null, "Expected a value of type {0} but property was an array: {1}", schemaPropertyDef.Type, inputProperty.Name));
return(PropertyValidationOutcome.InvalidType);
}
return(this.ValidateArrayProperty(inputProperty, schemas, detectedErrors, options));
}
else if (schemaPropertyDef.Type.IsObject && inputProperty.Type.IsObject)
{
// Compare the ODataType schema to the custom schema
if (null == schemaPropertyDef.Type.CustomTypeName || !schemas.ContainsKey(schemaPropertyDef.Type.CustomTypeName))
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.ResourceTypeNotFound, null, "Missing resource: resource {0} was not found (property name '{1}').", schemaPropertyDef.Type.CustomTypeName, inputProperty.Name));
return(PropertyValidationOutcome.MissingResourceType);
}
else if (inputProperty.Type.IsObject)
{
var odataSchema = schemas[schemaPropertyDef.Type.CustomTypeName];
ValidationError[] odataErrors;
if (null != inputProperty.Type.CustomMembers && !odataSchema.ValidateCustomObject(inputProperty.Type.CustomMembers.ToArray(), out odataErrors, schemas, options))
{
var propertyError = ValidationError.NewConsolidatedError(ValidationErrorCode.ConsolidatedError, odataErrors, "Schemas validation failed on property '{0}' ['{1}']", inputProperty.Name, odataSchema.ResourceName);
detectedErrors.Add(propertyError);
return(PropertyValidationOutcome.InvalidType);
}
else if (null == inputProperty.Type.CustomMembers)
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.NoCustomMembersFound, null, "Property '{0}' is of type Custom but has no custom members.", inputProperty.Name));
}
return(PropertyValidationOutcome.Ok);
}
else
{
var odataSchema = schemas[schemaPropertyDef.Type.CustomTypeName];
if (inputProperty.Type.CustomMembers == null)
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.MissingCustomMembers, null, "Property {0} is missing custom members and cannot be validated.", inputProperty.Name));
return(PropertyValidationOutcome.InvalidType);
}
else
{
odataSchema.ValidateObjectProperties(inputProperty.Type.CustomMembers, options, schemas, detectedErrors);
return(PropertyValidationOutcome.Ok);
}
}
}
else if (schemaPropertyDef.Type.IsObject)
{
detectedErrors.Add(new ValidationWarning(ValidationErrorCode.CustomValidationNotSupported, null, "Schemas type was 'Custom' which is not supported. Add a resource type to the definition of property: {0}", inputProperty.Name));
return(PropertyValidationOutcome.MissingResourceType);
}
else
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.ExpectedTypeDifferent, null, "Type mismatch: property '{0}' [{1}] doesn't match expected type [{2}].",
inputProperty.Name, inputProperty.Type, schemaPropertyDef.Type));
return(PropertyValidationOutcome.InvalidType);
}
}
else
{
// Check to see if this property is on the ignorable list
string[] ignorableUndocumentedProperties = this.OriginalResource?.SourceFile.Parent.Requirements?.IgnorableProperties;
string propertyName = inputProperty.Name;
string annotationName = null;
var indexOfAtSign = propertyName.IndexOf('@');
if (indexOfAtSign > 0)
{
// [email protected] is an example of what we're looking for here
annotationName = propertyName.Substring(indexOfAtSign);
propertyName = propertyName.Substring(0, indexOfAtSign);
}
if (null != annotationName)
{
// Check to see if propertyName is known or not. If it isn't known, fail.
if (this.Properties.Any(x => x.Name.Equals(propertyName)))
{
// If the cleaned up propertyName is known, then check to see if the annotation is ignorable
if (null != ignorableUndocumentedProperties && ignorableUndocumentedProperties.Contains(annotationName))
{
// If we know of both the property and the annotation, we're good.
return(PropertyValidationOutcome.Ok);
}
}
}
if (null != ignorableUndocumentedProperties && ignorableUndocumentedProperties.Contains(propertyName))
{
return(PropertyValidationOutcome.Ok);
}
// This property isn't documented
detectedErrors.Add(new UndocumentedPropertyWarning(null, inputProperty.Name, inputProperty.Type, ResourceName));
return(PropertyValidationOutcome.MissingFromSchema);
}
}
private void ValidateObjectProperties(IEnumerable<JsonProperty> propertiesOnObject, ValidationOptions options, Dictionary<string, JsonSchema> otherSchemas, List<ValidationError> detectedErrors)
{
List<string> missingProperties = new List<string>();
missingProperties.AddRange(from m in this.ExpectedProperties select m.Key);
foreach(var property in propertiesOnObject)
{
missingProperties.Remove(property.Name);
// This detects bad types, extra properties, etc.
if (null != options && (property.IsArray || property.Type == JsonDataType.ODataType || property.Type == JsonDataType.Object))
{
var propertyOptions = options.CreateForProperty(property.Name);
this.ValidateProperty(property, otherSchemas, detectedErrors, propertyOptions);
}
else
{
this.ValidateProperty(property, otherSchemas, detectedErrors, options);
}
}
this.CleanMissingProperties(options, missingProperties);
if (missingProperties.Count > 0)
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.RequiredPropertiesMissing, null, "Missing properties: response was missing these required properties: {0}", missingProperties.ComponentsJoinedByString(", ")));
}
}
/// <summary>
/// Check each member of the actualProperty's array to make sure it matches the resource type specified for the property.
/// </summary>
/// <param name="actualProperty"></param>
/// <param name="schemas"></param>
/// <param name="detectedErrors"></param>
/// <param name="options"></param>
private PropertyValidationOutcome ValidateArrayProperty(ParameterDefinition actualProperty, Dictionary <string, JsonSchema> schemas, List <ValidationError> detectedErrors, ValidationOptions options)
{
JArray actualArray = null;
try
{
actualArray = (JArray)JsonConvert.DeserializeObject(actualProperty.OriginalValue);
}
catch (InvalidCastException)
{
throw new InvalidCastException($"Property {actualProperty.Name} expected to be an array, but failed to cast value to an array: {actualProperty.OriginalValue}");
}
var expectedPropertyDefinition = this.ExpectedProperties[actualProperty.Name];
JsonSchema memberSchema = null;
if (actualProperty.Type.CollectionResourceType == SimpleDataType.Object && expectedPropertyDefinition.Type.CustomTypeName != null)
{
// We have an ambigious array, but we know what it's supposed to be so let's use that
schemas.TryGetValue(expectedPropertyDefinition.Type.CustomTypeName, out memberSchema);
}
if (memberSchema == null && string.IsNullOrEmpty(actualProperty.Type.CustomTypeName))
{
return(this.ValidateSimpleArrayProperty(actualProperty, this.ExpectedProperties[actualProperty.Name], detectedErrors));
}
else if (memberSchema == null && !schemas.TryGetValue(actualProperty.Type.CustomTypeName, out memberSchema))
{
detectedErrors.Add(new ValidationError(ValidationErrorCode.ResourceTypeNotFound, null, "Failed to locate resource definition for: {0}", actualProperty.Type.CustomTypeName));
return(PropertyValidationOutcome.MissingResourceType);
}
bool hadErrors = false;
for (int i = 0; i < actualArray.Count; i++)
{
JContainer member = actualArray[i] as JContainer;
if (member != null)
{
List <ValidationError> memberErrors = new List <ValidationError>();
memberSchema.ValidateContainerObject(member, options, schemas, memberErrors);
// TODO: Filter out non-unique errors
hadErrors |= memberErrors.Count > 0;
detectedErrors.AddUniqueErrors(memberErrors);
}
}
return(hadErrors ? PropertyValidationOutcome.GenericError : PropertyValidationOutcome.Ok);
}
/// <summary>
/// Validates the value of json according to an implicit schmea defined by expectedJson
/// </summary>
/// <param name="expectedResponseAnnotation"></param>
/// <param name="actualResponseBodyJson"></param>
/// <param name="errors"></param>
/// <returns></returns>
public bool ValidateJsonExample(CodeBlockAnnotation expectedResponseAnnotation, string actualResponseBodyJson, out ValidationError[] errors, ValidationOptions options = null)
{
List<ValidationError> newErrors = new List<ValidationError>();
var resourceType = expectedResponseAnnotation.ResourceType;
if (resourceType == "stream")
{
// No validation since we're streaming data
errors = null;
return true;
}
else
{
JsonSchema schema;
if (string.IsNullOrEmpty(resourceType))
{
schema = JsonSchema.EmptyResponseSchema;
}
else if (!this.registeredSchema.TryGetValue(resourceType, out schema))
{
newErrors.Add(new ValidationWarning(ValidationErrorCode.ResponseResourceTypeMissing, null, "Missing required resource: {0}. Validation limited to basics only.", resourceType));
// Create a new schema based on what's avaiable in the json
schema = new JsonSchema(actualResponseBodyJson, new CodeBlockAnnotation { ResourceType = expectedResponseAnnotation.ResourceType });
}
ValidationError[] validationJsonOutput;
this.ValidateJsonCompilesWithSchema(schema, new JsonExample(actualResponseBodyJson, expectedResponseAnnotation), out validationJsonOutput, options: options);
newErrors.AddRange(validationJsonOutput);
errors = newErrors.ToArray();
return errors.Length == 0;
}
}
private static async Task ValidateMethodWithScenarioAsync(
MethodDefinition method,
ScenarioDefinition scenario,
IServiceAccount account,
ValidationResults results,
ValidationOptions options = null)
{
if (null == method)
throw new ArgumentNullException("method");
if (null == scenario)
throw new ArgumentNullException("scenario");
if (null == account)
throw new ArgumentNullException("account");
if (null == results)
throw new ArgumentNullException("results");
var actionName = scenario.Description;
// Check to see if the account + scenario scopes are aligned
string[] requiredScopes = method.RequiredScopes.Union(scenario.RequiredScopes).ToArray();
if (!account.Scopes.ProvidesScopes(requiredScopes, options.IgnoreRequiredScopes))
{
var missingScopes = from scope in requiredScopes where !account.Scopes.Contains(scope) select scope;
results.AddResult(actionName,
new ValidationWarning(ValidationErrorCode.RequiredScopesMissing,
null,
"Scenario was not run. Scopes required were not available: {0}", missingScopes.ComponentsJoinedByString(",")));
return;
}
// Generate the tested request by "previewing" the request and executing
// all test-setup procedures
long startTicks = DateTimeOffset.UtcNow.Ticks;
var requestPreviewResult = await method.GenerateMethodRequestAsync(scenario, method.SourceFile.Parent, account);
TimeSpan generateMethodDuration = new TimeSpan(DateTimeOffset.UtcNow.Ticks - startTicks);
// Check to see if an error occured building the request, and abort if so.
var generatorResults = results[actionName /*+ " [test-setup requests]"*/];
generatorResults.AddResults(requestPreviewResult.Messages, requestPreviewResult.IsWarningOrError ? ValidationOutcome.Error : ValidationOutcome.Passed);
generatorResults.Duration = generateMethodDuration;
if (requestPreviewResult.IsWarningOrError)
{
return;
}
// We've done all the test-setup work, now we have the real request to make to the service
HttpRequest requestPreview = requestPreviewResult.Value;
results.AddResult(
actionName,
new ValidationMessage(null, "Generated Method HTTP Request:\r\n{0}", requestPreview.FullHttpText()));
HttpParser parser = new HttpParser();
HttpResponse expectedResponse = null;
if (!string.IsNullOrEmpty(method.ExpectedResponse))
{
expectedResponse = parser.ParseHttpResponse(method.ExpectedResponse);
}
// Execute the actual tested method (the result of the method preview call, which made the test-setup requests)
startTicks = DateTimeOffset.UtcNow.Ticks;
var actualResponse = await requestPreview.GetResponseAsync(account.BaseUrl);
TimeSpan actualMethodDuration = new TimeSpan(DateTimeOffset.UtcNow.Ticks - startTicks);
var requestResults = results[actionName];
if (actualResponse.RetryCount > 0)
{
requestResults.AddResults(
new ValidationError[]
{ new ValidationWarning(ValidationErrorCode.RequestWasRetried, null, "HTTP request was retried {0} times.", actualResponse.RetryCount) });
}
requestResults.AddResults(
new ValidationError[]
{ new ValidationMessage(null, "HTTP Response:\r\n{0}", actualResponse.FullText(false)) });
requestResults.Duration = actualMethodDuration;
// Perform validation on the method's actual response
ValidationError[] errors;
method.ValidateResponse(actualResponse, expectedResponse, scenario, out errors, options);
requestResults.AddResults(errors);
// TODO: If the method is defined as a long running operation, we need to go poll the status
// URL to make sure that the operation finished and the response type is valid.
if (errors.WereErrors())
results.SetOutcome(actionName, ValidationOutcome.Error);
else if (errors.WereWarnings())
results.SetOutcome(actionName, ValidationOutcome.Warning);
else
results.SetOutcome(actionName, ValidationOutcome.Passed);
}
