/// <summary>
/// Validates the value of json according to an implicit schmea defined by expectedJson
/// </summary>
/// <returns></returns>
public bool ValidateJsonExample(CodeBlockAnnotation expectedResponseAnnotation, string actualResponseBodyJson, IssueLogger issues, ValidationOptions options = null)
{
List <ValidationError> newErrors = new List <ValidationError>();
var resourceType = expectedResponseAnnotation.ResourceType;
if (resourceType == "stream")
{
// No validation since we're streaming data
return(true);
}
else
{
JsonSchema schema;
if (string.IsNullOrEmpty(resourceType))
{
schema = JsonSchema.EmptyResponseSchema;
}
else if (!this.registeredSchema.TryGetValue(resourceType, out schema))
{
newErrors.Add(new ValidationWarning(ValidationErrorCode.ResponseResourceTypeMissing, null, "Missing required resource: {0}. Validation limited to basics only.", resourceType));
// Create a new schema based on what's available in the json
schema = new JsonSchema(actualResponseBodyJson, new CodeBlockAnnotation {
ResourceType = expectedResponseAnnotation.ResourceType
});
}
this.ValidateJsonCompilesWithSchema(schema, new JsonExample(actualResponseBodyJson, expectedResponseAnnotation), issues, options: options);
return(issues.Issues.Count() == 0);
}
}
/// <summary>
/// Validates that the actual response body matches the schema defined for the response and any additional constraints
/// from the expected request (e.g. properties that are included in the expected response are required in the actual
/// response even if the metadata defines that the response is truncated)
/// </summary>
internal bool ValidateResponseMatchesSchema(MethodDefinition method, HttpResponse actualResponse, HttpResponse expectedResponse, IssueLogger issues, ValidationOptions options = null)
{
var expectedResourceType = method.ExpectedResponseMetadata.ResourceType;
switch (expectedResourceType)
{
case "stream":
case "Stream":
case "Edm.stream":
case "Edm.Stream":
// No validation since we're streaming data
return(true);
case "string":
case "String":
case "Edm.String":
case " Edm.string":
return(true);
}
// Get a reference of our JsonSchema that we're checking the response with
var expectedResponseJson = (null != expectedResponse) ? expectedResponse.Body : null;
JsonSchema schema = this.GetJsonSchema(expectedResourceType, issues, expectedResponseJson);
if (null == schema)
{
issues.Error(ValidationErrorCode.ResourceTypeNotFound, $"Unable to locate a definition for resource type: {expectedResourceType}");
}
else
{
this.ValidateJsonCompilesWithSchema(schema, new JsonExample(actualResponse.Body, method.ExpectedResponseMetadata), issues, (null != expectedResponseJson) ? new JsonExample(expectedResponseJson) : null, options);
}
if (issues.Issues.WereWarningsOrErrors() && method.ExpectedResponseMetadata.ResourceTypeAka != null)
{
var expectedResourceTypeAka = method.ExpectedResponseMetadata.ResourceTypeAka;
schema = this.GetJsonSchema(expectedResourceTypeAka, issues, expectedResponseJson);
if (null == schema)
{
issues.Error(ValidationErrorCode.ResourceTypeNotFound, $"Unable to locate a definition for resource type: {expectedResourceTypeAka}");
}
else
{
this.ValidateJsonCompilesWithSchema(schema, new JsonExample(actualResponse.Body, method.ExpectedResponseMetadata), issues, (null != expectedResponseJson) ? new JsonExample(expectedResponseJson) : null, options);
}
}
return(!issues.Issues.WereWarningsOrErrors());
}
private bool ValidateCustomObject(ParameterDefinition[] properties, IssueLogger issues, Dictionary <string, JsonSchema> otherSchemas, ValidationOptions options)
{
List <string> missingProperties = new List <string>(this.ExpectedProperties.Keys);
foreach (var inputProperty in properties)
{
missingProperties.Remove(inputProperty.Name);
this.ValidateProperty(inputProperty, otherSchemas, issues.For(inputProperty.Name), options);
}
this.CleanMissingProperties(options, missingProperties);
return(!issues.Issues.WereWarningsOrErrors());
}
/// <summary>
/// Examines input json string to ensure that it compiles with the JsonSchema definition. Any errors in the
/// validation of the schema are returned via the errors out parameter.
/// </summary>
/// <param name="schema">Schemas definition used as a reference.</param>
/// <param name="inputJson">Input json example to be validated</param>
/// <param name="issues"></param>
/// <param name="expectedJson"></param>
/// <returns></returns>
public bool ValidateJsonCompilesWithSchema(JsonSchema schema, JsonExample inputJson, IssueLogger issues, JsonExample expectedJson = null, ValidationOptions options = null)
{
if (null == schema)
{
throw new ArgumentNullException("schema");
}
if (null == inputJson)
{
throw new ArgumentNullException("inputJson");
}
string collectionPropertyName = "value";
if (null != inputJson.Annotation && null != inputJson.Annotation.CollectionPropertyName)
{
collectionPropertyName = inputJson.Annotation.CollectionPropertyName;
}
// If we didn't get an options, create a new one with some defaults provided by the annotation
options = options ?? new ValidationOptions();
var annotationTruncated = (inputJson.Annotation ?? new CodeBlockAnnotation()).TruncatedResult;
options.AllowTruncatedResponses = annotationTruncated || options.AllowTruncatedResponses;
options.CollectionPropertyName = collectionPropertyName;
return(schema.ValidateJson(inputJson, issues, this.registeredSchema, options, expectedJson));
}
/// <summary>
/// Check each member of the actualProperty's array to make sure it matches the resource type specified for the property.
/// </summary>
/// <param name="actualProperty"></param>
/// <param name="schemas"></param>
/// <param name="detectedErrors"></param>
/// <param name="options"></param>
private PropertyValidationOutcome ValidateArrayProperty(ParameterDefinition actualProperty, Dictionary <string, JsonSchema> schemas, IssueLogger issues, ValidationOptions options)
{
JArray actualArray = null;
try
{
actualArray = (JArray)JsonConvert.DeserializeObject(actualProperty.OriginalValue);
}
catch (InvalidCastException)
{
throw new InvalidCastException($"Property {actualProperty.Name} expected to be an array, but failed to cast value to an array: {actualProperty.OriginalValue}");
}
var expectedPropertyDefinition = this.ExpectedProperties[actualProperty.Name];
JsonSchema memberSchema = null;
if (actualProperty.Type.CollectionResourceType == SimpleDataType.Object && expectedPropertyDefinition.Type.CustomTypeName != null)
{
// We have an ambigious array, but we know what it's supposed to be so let's use that
schemas.TryGetValue(expectedPropertyDefinition.Type.CustomTypeName, out memberSchema);
}
if (memberSchema == null && string.IsNullOrEmpty(actualProperty.Type.CustomTypeName))
{
return(this.ValidateSimpleArrayProperty(actualProperty, this.ExpectedProperties[actualProperty.Name], issues));
}
else if (memberSchema == null && !schemas.TryGetValue(actualProperty.Type.CustomTypeName, out memberSchema))
{
issues.Error(ValidationErrorCode.ResourceTypeNotFound, $"Failed to locate resource definition for: {actualProperty.Type.CustomTypeName}");
return(PropertyValidationOutcome.MissingResourceType);
}
bool hadErrors = false;
var memberIssues = issues.For("member", onlyKeepUniqueErrors: true);
for (int i = 0; i < actualArray.Count; i++)
{
JContainer member = actualArray[i] as JContainer;
if (member != null)
{
memberSchema.ValidateContainerObject(member, options, schemas, memberIssues);
// TODO: Filter out non-unique errors
hadErrors |= memberIssues.Issues.Count() > 0;
}
}
return(hadErrors ? PropertyValidationOutcome.GenericError : PropertyValidationOutcome.Ok);
}
/// <summary>
/// Verify that a property from the json-to-validate matches something in our schema
/// </summary>
private PropertyValidationOutcome ValidateProperty(ParameterDefinition inputProperty, Dictionary <string, JsonSchema> schemas, IssueLogger issues, ValidationOptions options)
{
if (this.ExpectedProperties.ContainsKey(inputProperty.Name))
{
// The property was expected to be found in this schema! Yay.
var schemaPropertyDef = this.ExpectedProperties[inputProperty.Name];
// Check for simple value types first
if (this.SimpleValueTypes(schemaPropertyDef.Type, inputProperty.Type) && this.AllFalse(schemaPropertyDef.Type.IsCollection, inputProperty.Type.IsCollection))
{
return(ValidateSameDataType(schemaPropertyDef, inputProperty, issues, (null != options) ? options.RelaxedStringValidation : false));
}
else if (null == inputProperty.OriginalValue)
{
if (null != this.NullableProperties && !this.NullableProperties.Contains(schemaPropertyDef.Name))
{
issues.Warning(ValidationErrorCode.NullPropertyValue, $"Non-nullable property {schemaPropertyDef.Name} had a null value in the response. Expected {schemaPropertyDef.Type}.");
}
return(PropertyValidationOutcome.Ok);
}
else if (schemaPropertyDef.Type.IsCollection || inputProperty.Type.IsCollection)
{
// Check for an array
if (schemaPropertyDef.Type.IsCollection && !inputProperty.Type.IsCollection)
{
// Expected an array, but didn't get one
issues.Error(ValidationErrorCode.ExpectedArrayValue, $"Expected an array but property was not an array: {inputProperty.Name}");
return(PropertyValidationOutcome.InvalidType);
}
else if (!schemaPropertyDef.Type.IsCollection && inputProperty.Type.IsCollection)
{
issues.Error(ValidationErrorCode.ExpectedNonArrayValue, $"Expected a value of type {schemaPropertyDef.Type} but property was an array: {inputProperty.Name}");
return(PropertyValidationOutcome.InvalidType);
}
return(this.ValidateArrayProperty(inputProperty, schemas, issues, options));
}
else if (schemaPropertyDef.Type.IsObject && inputProperty.Type.IsObject)
{
// Compare the ODataType schema to the custom schema
if (null == schemaPropertyDef.Type.CustomTypeName || !schemas.ContainsKey(schemaPropertyDef.Type.CustomTypeName))
{
issues.Error(ValidationErrorCode.ResourceTypeNotFound, $"Missing resource: resource {schemaPropertyDef.Type.CustomTypeName} was not found (property name '{inputProperty.Name}').");
return(PropertyValidationOutcome.MissingResourceType);
}
else if (inputProperty.Type.IsObject)
{
var odataSchema = schemas[schemaPropertyDef.Type.CustomTypeName];
if (null != inputProperty.Type.CustomMembers && !odataSchema.ValidateCustomObject(inputProperty.Type.CustomMembers.ToArray(), issues, schemas, options))
{
if (issues.Errors.Any())
{
issues.Error(ValidationErrorCode.ConsolidatedError, $"Schema validation failed on property '{inputProperty.Name}' ['{odataSchema.ResourceName}']");
}
else
{
issues.Warning(ValidationErrorCode.ConsolidatedError, $"Schema validation failed on property '{inputProperty.Name}' ['{odataSchema.ResourceName}']");
}
return(PropertyValidationOutcome.InvalidType);
}
else if (null == inputProperty.Type.CustomMembers)
{
issues.Error(ValidationErrorCode.NoCustomMembersFound, $"Property '{inputProperty.Name}' is of type Custom but has no custom members.");
}
return(PropertyValidationOutcome.Ok);
}
else
{
var odataSchema = schemas[schemaPropertyDef.Type.CustomTypeName];
if (inputProperty.Type.CustomMembers == null)
{
issues.Error(ValidationErrorCode.MissingCustomMembers, $"Property {inputProperty.Name} is missing custom members and cannot be validated.");
return(PropertyValidationOutcome.InvalidType);
}
else
{
odataSchema.ValidateObjectProperties(inputProperty.Type.CustomMembers, options, schemas, issues);
return(PropertyValidationOutcome.Ok);
}
}
}
else if (schemaPropertyDef.Type.IsObject)
{
issues.Warning(ValidationErrorCode.CustomValidationNotSupported, $"Schemas type was 'Custom' which is not supported. Add a resource type to the definition of property: {inputProperty.Name}");
return(PropertyValidationOutcome.MissingResourceType);
}
else
{
issues.Error(ValidationErrorCode.ExpectedTypeDifferent, $"Type mismatch: property '{inputProperty.Name}' [{inputProperty.Type}] doesn't match expected type [{schemaPropertyDef.Type}].");
return(PropertyValidationOutcome.InvalidType);
}
}
else
{
// Check to see if this property is on the ignorable list
string[] ignorableUndocumentedProperties = this.OriginalResource?.SourceFile.Parent.Requirements?.IgnorableProperties;
string propertyName = inputProperty.Name;
string annotationName = null;
var indexOfAtSign = propertyName.IndexOf('@');
if (indexOfAtSign > 0)
{
// [email protected] is an example of what we're looking for here
annotationName = propertyName.Substring(indexOfAtSign);
propertyName = propertyName.Substring(0, indexOfAtSign);
}
if (null != annotationName)
{
// Check to see if propertyName is known or not. If it isn't known, fail.
if (this.Properties.Any(x => x.Name.Equals(propertyName)))
{
// If the cleaned up propertyName is known, then check to see if the annotation is ignorable
if (null != ignorableUndocumentedProperties && ignorableUndocumentedProperties.Contains(annotationName))
{
// If we know of both the property and the annotation, we're good.
return(PropertyValidationOutcome.Ok);
}
}
}
if (null != ignorableUndocumentedProperties && ignorableUndocumentedProperties.Contains(propertyName))
{
return(PropertyValidationOutcome.Ok);
}
if (this.OriginalResource?.OriginalMetadata?.IsOpenType == true)
{
return(PropertyValidationOutcome.Ok);
}
// This property isn't documented
issues.Warning(new UndocumentedPropertyWarning(null, inputProperty.Name, inputProperty.Type, ResourceName));
return(PropertyValidationOutcome.MissingFromSchema);
}
}
private void ValidateObjectProperties(IEnumerable <ParameterDefinition> propertiesOnObject, ValidationOptions options, Dictionary <string, JsonSchema> otherSchemas, IssueLogger issues)
{
List <string> missingProperties = new List <string>();
missingProperties.AddRange(from m in this.ExpectedProperties select m.Key);
foreach (var property in propertiesOnObject)
{
missingProperties.Remove(property.Name);
// This detects bad types, extra properties, etc.
if (null != options && (property.Type.IsCollection || property.Type.IsObject))
{
var propertyOptions = options.CreateForProperty(property.Name);
this.ValidateProperty(property, otherSchemas, issues.For(property.Name), propertyOptions);
}
else
{
this.ValidateProperty(property, otherSchemas, issues.For(property.Name), options);
}
}
this.CleanMissingProperties(options, missingProperties);
if (missingProperties.Count > 0)
{
issues.Error(ValidationErrorCode.RequiredPropertiesMissing,
$"Missing properties: response was missing these required properties: {missingProperties.ComponentsJoinedByString(", ")}");
}
}
private void ValidateCollectionObject(JContainer obj, CodeBlockAnnotation annotation, Dictionary <string, JsonSchema> otherSchemas, string collectionPropertyName, IssueLogger issues, ValidationOptions options)
{
// TODO: also validate additional properties on the collection, like nextDataLink
JToken collection = null;
if (obj is JArray)
{
collection = obj;
}
else
{
try
{
collection = obj[collectionPropertyName];
}
catch (Exception ex)
{
issues.Error(ValidationErrorCode.JsonErrorObjectExpected, $"Unable to find collection parameter or array to validate: {ex.Message}");
return;
}
}
if (null == collection)
{
issues.Error(ValidationErrorCode.MissingCollectionProperty, $"Failed to locate collection property '{collectionPropertyName}' in response.");
}
else
{
if (!collection.Any())
{
if (!annotation.IsEmpty)
{
issues.Warning(
ValidationErrorCode.CollectionArrayEmpty,
$"Property contained an empty array that was not validated: {collectionPropertyName}");
}
}
else if (annotation.IsEmpty)
{
issues.Warning(
ValidationErrorCode.CollectionArrayNotEmpty,
$"Property contained a non-empty array that was expected to be empty: {collectionPropertyName}");
}
foreach (var jToken in collection)
{
var container = jToken as JContainer;
if (null != container)
{
var deeperOptions = new ValidationOptions(options)
{
AllowTruncatedResponses = annotation.TruncatedResult
};
this.ValidateContainerObject(
container,
deeperOptions,
otherSchemas,
issues.For("container", onlyKeepUniqueErrors: true));
}
}
}
}
/// <summary>
/// Validate the input json against the defined scehma when the instance was created.
/// </summary>
/// <param name="jsonInput">Input json to validate against schema</param>
/// <param name="issues"></param>
/// <param name="otherSchemas"></param>
/// <param name="options"></param>
/// <param name="expectedJson"></param>
/// <returns>True if validation was successful, otherwise false.</returns>
public bool ValidateJson(JsonExample jsonInput, IssueLogger issues, Dictionary <string, JsonSchema> otherSchemas, ValidationOptions options, JsonExample expectedJson = null)
{
JContainer obj;
try
{
var settings = new JsonSerializerSettings {
DateParseHandling = DateParseHandling.None, NullValueHandling = NullValueHandling.Include, DefaultValueHandling = DefaultValueHandling.Include
};
obj = (JContainer)JsonConvert.DeserializeObject(jsonInput.JsonData, settings);
}
catch (Exception ex)
{
issues.Error(ValidationErrorCode.JsonParserException, $"Failed to parse json string: {jsonInput.JsonData}.", ex);
return(false);
}
var annotation = jsonInput.Annotation ?? new CodeBlockAnnotation();
bool expectErrorObject = (jsonInput.Annotation != null) && jsonInput.Annotation.ExpectError;
// Check for an error response
try
{
dynamic errorObject = obj["error"];
if (null != errorObject && !expectErrorObject)
{
string code = errorObject.code;
string message = errorObject.message;
issues.Error(ValidationErrorCode.JsonErrorObject, $"Error response received. Code: {code}, Message: {message}");
return(false);
}
else if (expectErrorObject && null == errorObject)
{
issues.Error(ValidationErrorCode.JsonErrorObjectExpected, "Expected an error object response, but didn't receive one.");
return(false);
}
}
catch (Exception ex)
{
if (annotation.ExpectError)
{
issues.Error(ValidationErrorCode.JsonErrorObjectExpected, "Expected an error object, but it doesn't look like we got one.", ex);
}
}
// Check to see if this is a "collection" instance
if (null != annotation && annotation.IsCollection)
{
this.ValidateCollectionObject(obj, annotation, otherSchemas, options.CollectionPropertyName, issues, options);
}
// otherwise verify the object matches this schema
else
{
options = options ?? new ValidationOptions(annotation);
if (null != expectedJson)
{
var expectedJsonSchema = new JsonSchema(expectedJson.JsonData, expectedJson.Annotation);
options.ExpectedJsonSchema = expectedJsonSchema;
options.RequiredPropertyNames = expectedJsonSchema.ExpectedProperties.Keys.ToArray();
}
this.ValidateContainerObject(obj, options, otherSchemas, issues);
}
return(issues.Issues.Count() == 0);
}