public void Failures_WithNestedChildResultsWithFailures_ReturnsAllFailuresFromNestedResultsInCorrectOrder() { var failure1 = "Failure 1"; var failure2 = "Failure 2"; var failure3 = "Failure 3"; var nestedChildResult2 = new ComparisonResult(); nestedChildResult2.RecordFailure(new ErrorMessageComparisonFailure(failure3)); var nestedChildResult = new ComparisonResult(); nestedChildResult.RecordFailure(new ErrorMessageComparisonFailure(failure2)); nestedChildResult.AddChildResult(nestedChildResult2); var comparisonResult = GetSubject(); comparisonResult.RecordFailure(new ErrorMessageComparisonFailure(failure1)); comparisonResult.AddChildResult(nestedChildResult); var failures = comparisonResult.Failures; Assert.Equal(3, failures.Count()); Assert.Equal(failure1, failures.First().Result); Assert.Equal(failure2, failures.Skip(1).First().Result); Assert.Equal(failure3, failures.Last().Result); }
public ComparisonResult Compare(ProviderServiceRequest expected, ProviderServiceRequest actual) { var result = new ComparisonResult("sends a request which"); if (expected == null) { result.RecordFailure(new ErrorMessageComparisonFailure("Expected request cannot be null")); return result; } var methodResult = _httpMethodComparer.Compare(expected.Method, actual.Method); result.AddChildResult(methodResult); var pathResult = _httpPathComparer.Compare(expected.Path, actual.Path); result.AddChildResult(pathResult); var queryResult = _httpQueryStringComparer.Compare(expected.Query, actual.Query); result.AddChildResult(queryResult); if (expected.Headers != null && expected.Headers.Any()) { var headerResult = _httpHeaderComparer.Compare(expected.Headers, actual.Headers); result.AddChildResult(headerResult); } if (expected.Body != null) { var bodyResult = _httpBodyComparer.Compare(expected.Body, actual.Body, expected.MatchingRules); result.AddChildResult(bodyResult); } return result; }
public ComparisonResult Compare(ProviderServiceResponse expected, ProviderServiceResponse actual) { var result = new ComparisonResult("returns a response which"); if (expected == null) { result.RecordFailure(new ErrorMessageComparisonFailure("Expected response cannot be null")); return result; } var statusResult = _httpStatusCodeComparer.Compare(expected.Status, actual.Status); result.AddChildResult(statusResult); if (expected.Headers != null && expected.Headers.Any()) { var headerResult = _httpHeaderComparer.Compare(expected.Headers, actual.Headers); result.AddChildResult(headerResult); } if (expected.Body != null) { var bodyResult = _httpBodyComparer.Compare(expected.Body, actual.Body, expected.MatchingRules); result.AddChildResult(bodyResult); } else if (expected.Body == null && expected.ShouldSerializeBody() && //Body was explicitly set actual.Body != null) { result.RecordFailure(new ErrorMessageComparisonFailure("Expected response body was explicitly set to null however the actual response body was not null")); } return result; }
public void AddChildResult(ComparisonResult comparisonResult) { if (comparisonResult == null) { return; } _childResults.Add(comparisonResult); }
public ComparisonResult Compare(ProviderServiceRequest expected, ProviderServiceRequest actual) { var result = new ComparisonResult("sends a request which"); if (expected == null) { result.RecordFailure(new ErrorMessageComparisonFailure("Expected request cannot be null")); return result; } var methodResult = _httpMethodComparer.Compare(expected.Method, actual.Method); result.AddChildResult(methodResult); var pathResult = _httpPathComparer.Compare(expected.Path, actual.Path); result.AddChildResult(pathResult); var queryResult = _httpQueryStringComparer.Compare(expected.Query, actual.Query); result.AddChildResult(queryResult); if (expected.Headers != null && expected.Headers.Any()) { var headerResult = _httpHeaderComparer.Compare(expected.Headers, actual.Headers); result.AddChildResult(headerResult); } if (expected.Body != null || actual.Body != null) { if (expected.Body == null) { result.RecordFailure(new ErrorMessageComparisonFailure( "Expected request does not match actual request")); return result; } if (actual.Body == null) { result.RecordFailure(new ErrorMessageComparisonFailure( "Expected request does not match actual request")); return result; } var expectedToken2 = JToken.FromObject(expected.Body); var actualToken2 = JToken.FromObject(actual.Body); if (!TestUtils.CheckAllPropertiesAreEqual(expectedToken2, actualToken2, expected.IgnoreList)) { result.RecordFailure( new ErrorMessageComparisonFailure("Expected request does not match actual request")); } } return result; }
public ComparisonResult Compare(HttpVerb expected, HttpVerb actual) { var result = new ComparisonResult("has method {0}", expected); if (!expected.Equals(actual)) { result.RecordFailure(new DiffComparisonFailure(expected, actual)); return result; } return result; }
public ComparisonResult Compare(IDictionary<string, string> expected, IDictionary<string, string> actual) { var result = new ComparisonResult("includes headers"); if (actual == null) { result.RecordFailure(new ErrorMessageComparisonFailure("Actual Headers are null")); return result; } actual = MakeDictionaryCaseInsensitive(actual); foreach (var header in expected) { var headerResult = new ComparisonResult("'{0}' with value {1}", header.Key, header.Value); string actualValue; if (actual.TryGetValue(header.Key, out actualValue)) { var expectedValue = header.Value; var actualValueSplit = actualValue.Split(new[] { ',', ';' }); if (actualValueSplit.Length == 1) { if (!header.Value.Equals(actualValue)) { headerResult.RecordFailure(new DiffComparisonFailure(expectedValue, actualValue)); } } else { var expectedValueSplit = expectedValue.Split(new[] {',', ';'}); var expectedValueSplitJoined = String.Join(",", expectedValueSplit.Select(x => x.Trim())); var actualValueSplitJoined = String.Join(",", actualValueSplit.Select(x => x.Trim())); if (!expectedValueSplitJoined.Equals(actualValueSplitJoined)) { headerResult.RecordFailure(new DiffComparisonFailure(expectedValue, actualValue)); } } } else { headerResult.RecordFailure(new ErrorMessageComparisonFailure(String.Format("Header with key '{0}', does not exist in actual", header.Key))); } result.AddChildResult(headerResult); } return result; }
public void ShallowFailureCount_WithOnlyNestedChildResultsWithFailures_ReturnsZero() { var nestedChildResult = new ComparisonResult(); nestedChildResult.RecordFailure(new ErrorMessageComparisonFailure("failure 1")); var comparisonResult = GetSubject(); comparisonResult.AddChildResult(nestedChildResult); var shallowFailureCount = comparisonResult.ShallowFailureCount; Assert.Equal(0, shallowFailureCount); }
public void HasFailure_WithOnlyNestedChildResultsWithFailures_ReturnsTrue() { var nestedChildResult = new ComparisonResult(); nestedChildResult.RecordFailure(new ErrorMessageComparisonFailure("failure 1")); var comparisonResult = GetSubject(); comparisonResult.AddChildResult(nestedChildResult); var hasFailure = comparisonResult.HasFailure; Assert.True(hasFailure); }
public void ReportFailureReasons_WithNoFailuresOnComparisonResult_DoesNotCallTheOutputter() { const string comparisonMessage = "The thing I am testing"; var reporter = GetSubject(); var comparisonResult = new ComparisonResult(comparisonMessage); reporter.ReportFailureReasons(comparisonResult); _mockOutputter.DidNotReceive().WriteInfo(Arg.Any<String>(), Arg.Any<int>()); _mockOutputter.DidNotReceive().WriteError(Arg.Any<String>(), Arg.Any<int>()); }
public ComparisonResult Compare(string expected, string actual) { if (String.IsNullOrEmpty(expected) && String.IsNullOrEmpty(actual)) { return new ComparisonResult("has no query strings"); } var normalisedExpectedQuery = NormaliseUrlEncodingAndTrimTrailingAmpersand(expected); var normalisedActualQuery = NormaliseUrlEncodingAndTrimTrailingAmpersand(actual); var result = new ComparisonResult("has query {0}", normalisedExpectedQuery ?? "null"); if (expected == null || actual == null) { result.RecordFailure(new DiffComparisonFailure(expected, actual)); return result; } var expectedQueryItems = HttpUtility.ParseQueryString(normalisedExpectedQuery); var actualQueryItems = HttpUtility.ParseQueryString(normalisedActualQuery); if (expectedQueryItems.Count != actualQueryItems.Count) { result.RecordFailure(new DiffComparisonFailure(normalisedExpectedQuery, normalisedActualQuery)); return result; } foreach (string expectedKey in expectedQueryItems) { if (!actualQueryItems.AllKeys.Contains(expectedKey)) { result.RecordFailure(new DiffComparisonFailure(normalisedExpectedQuery, normalisedActualQuery)); return result; } if (actualQueryItems.AllKeys.Contains(expectedKey)) { var expectedValue = expectedQueryItems[expectedKey]; var actualValue = actualQueryItems[expectedKey]; var isWildcard = IsWildcard(expectedValue); if (!isWildcard && expectedValue != actualValue) { result.RecordFailure(new DiffComparisonFailure(normalisedExpectedQuery, normalisedActualQuery)); return result; } } } return result; }
public void Flush_WithAReportedSummaryThatContainsFailuresOnTheComparisonResult_CallsOutputterWithMessage() { const string comparisonMessage = "The thing I am testing"; var expectedMessage = String.Format("{0} (FAILED - 1)", comparisonMessage); var reporter = GetSubject(); var comparisonResult = new ComparisonResult(comparisonMessage); comparisonResult.RecordFailure(new ErrorMessageComparisonFailure("It failed")); reporter.ReportSummary(comparisonResult); reporter.Flush(); _reportOutputter.Received(1).Write(expectedMessage); }
public ComparisonResult Compare(string expected, string actual) { var result = new ComparisonResult("has path {0}", expected); if (expected == null) { return result; } if (!expected.Equals(actual)) { result.RecordFailure(new DiffComparisonFailure(expected, actual)); return result; } return result; }
public void ReportFailureReasons_WithFailuresOnComparisonResult_CallsWriteErrorOnOutputterWithFailures() { const string comparisonMessage = "The thing I am testing"; const string comparisonFailureMessage1 = "It failed 1"; const string comparisonFailureMessage2 = "It failed 2"; var reporter = GetSubject(); var comparisonResult = new ComparisonResult(comparisonMessage); comparisonResult.RecordFailure(new ErrorMessageComparisonFailure(comparisonFailureMessage1)); comparisonResult.RecordFailure(new ErrorMessageComparisonFailure(comparisonFailureMessage2)); reporter.ReportFailureReasons(comparisonResult); _mockOutputter.Received(1).WriteError(Environment.NewLine + "1) " + comparisonFailureMessage1, Arg.Any<int>()); _mockOutputter.Received(1).WriteError(Environment.NewLine + "2) " + comparisonFailureMessage2, Arg.Any<int>()); }
public ComparisonResult Compare(dynamic expected, dynamic actual, IDictionary<string, IMatcher> matchingRules) { var result = new ComparisonResult("has a matching body"); if (expected == null) { return result; } if (expected != null && actual == null) { result.RecordFailure(new ErrorMessageComparisonFailure("Actual Body is null")); return result; } var expectedToken = JToken.FromObject(expected); var actualToken = JToken.FromObject(actual); foreach (var rule in matchingRules) { MatcherResult matchResult = rule.Value.Match(rule.Key, expectedToken, actualToken); //TODO: Maybe we should call this a list of differences var comparisonFailures = new List<ComparisonFailure>(); foreach (var failedCheck in matchResult.MatcherChecks.Where(x => x is FailedMatcherCheck).Cast<FailedMatcherCheck>()) { //TODO: We should be able to generate a better output, as we know exactly the path that failed var comparisonFailure = new DiffComparisonFailure(expectedToken, actualToken); if (comparisonFailures.All(x => x.Result != comparisonFailure.Result)) { comparisonFailures.Add(comparisonFailure); } } foreach (var failure in comparisonFailures) { result.RecordFailure(failure); } //TODO: When more than 1 rule deal with the situation when a success overrides a failure (either more specific rule or order it's applied?) } return result; }
private void WriteFailureReasons(ComparisonResult comparisonResult) { if (comparisonResult == null) { return; } if (!comparisonResult.HasFailure) { return; } _outputter.WriteInfo(Environment.NewLine + "Failures:"); foreach (var failure in comparisonResult.Failures) { _outputter.WriteError(String.Format("{0}{1}) {2}", Environment.NewLine, ++_failureCount, failure.Result)); } }
public void ReportSummary_WithChildResultMultipleFailuresOnComparisonResult_CallsWriteErrorOnOutputterWithMessage() { const string comparisonMessage1 = "The thing I am testing"; const string comparisonMessage2 = "The thing I am testing 2"; var reporter = GetSubject(); var comparisonResult2 = new ComparisonResult(comparisonMessage2); comparisonResult2.RecordFailure(new ErrorMessageComparisonFailure("Failure 2")); var comparisonResult = new ComparisonResult(comparisonMessage1); comparisonResult.RecordFailure(new ErrorMessageComparisonFailure("Failure 1")); comparisonResult.AddChildResult(comparisonResult2); reporter.ReportSummary(comparisonResult); _mockOutputter.Received(1).WriteError(comparisonMessage1 + " (FAILED - 1)", Arg.Any<int>()); _mockOutputter.Received(1).WriteError(comparisonMessage2 + " (FAILED - 2)", Arg.Any<int>()); }
public void Flush_WithAReportedSummaryThatContainsAChildResultWithMultipleFailuresOnTheComparisonResult_CallsOutputterWithMessage() { const string comparisonMessage1 = "The thing I am testing"; const string comparisonMessage2 = "The thing I am testing 2"; var reporter = GetSubject(); var comparisonResult2 = new ComparisonResult(comparisonMessage2); comparisonResult2.RecordFailure(new ErrorMessageComparisonFailure("Failure 2")); var comparisonResult = new ComparisonResult(comparisonMessage1); comparisonResult.RecordFailure(new ErrorMessageComparisonFailure("Failure 1")); comparisonResult.AddChildResult(comparisonResult2); reporter.ReportSummary(comparisonResult); reporter.Flush(); _reportOutputter.Received(1).Write(Arg.Is<string>(x => x.Contains(comparisonMessage1 + " (FAILED - 1)") && x.Contains(comparisonMessage2 + " (FAILED - 2)"))); }
private void WriteSummary(ComparisonResult comparisonResult, int tabDepth = 0) { if (comparisonResult == null) { return; } if (comparisonResult.HasFailure) { var failureBuilder = new StringBuilder(); var shallowFailureCount = comparisonResult.ShallowFailureCount; if (shallowFailureCount > 0) { failureBuilder.Append(" (FAILED - "); for (var i = 0; i < shallowFailureCount; i++) { failureBuilder.Append(++_failureInfoCount); if (i < shallowFailureCount - 1) { failureBuilder.Append(", "); } } failureBuilder.Append(")"); } _outputter.WriteError(comparisonResult.Message + failureBuilder, _currentTabDepth + tabDepth); } else { _outputter.WriteSuccess(comparisonResult.Message, _currentTabDepth + tabDepth); } foreach (var childComparisonResult in comparisonResult.ChildResults) { WriteSummary(childComparisonResult, tabDepth + 1); } }
public void Validate(ProviderServicePactFile pactFile, ProviderStates providerStates) { if (pactFile == null) { throw new ArgumentException("Please supply a non null pactFile"); } if (pactFile.Consumer == null || String.IsNullOrEmpty(pactFile.Consumer.Name)) { throw new ArgumentException("Please supply a non null or empty Consumer name in the pactFile"); } if (pactFile.Provider == null || String.IsNullOrEmpty(pactFile.Provider.Name)) { throw new ArgumentException("Please supply a non null or empty Provider name in the pactFile"); } if (pactFile.Interactions != null && pactFile.Interactions.Any()) { _reporter.ReportInfo(String.Format("Verifying a Pact between {0} and {1}", pactFile.Consumer.Name, pactFile.Provider.Name)); var comparisonResult = new ComparisonResult(); foreach (var interaction in pactFile.Interactions) { InvokePactSetUpIfApplicable(providerStates); _reporter.ResetIndentation(); ProviderState providerStateItem = null; if (interaction.ProviderState != null) { try { providerStateItem = providerStates.Find(interaction.ProviderState); } catch (Exception) { providerStateItem = null; } if (providerStateItem == null) { throw new InvalidOperationException(String.Format("providerState '{0}' was defined by a consumer, however could not be found. Please supply this provider state.", interaction.ProviderState)); } } InvokeProviderStateSetUpIfApplicable(providerStateItem); if (!String.IsNullOrEmpty(interaction.ProviderState)) { _reporter.Indent(); _reporter.ReportInfo(String.Format("Given {0}", interaction.ProviderState)); } _reporter.Indent(); _reporter.ReportInfo(String.Format("{0}", interaction.Description)); if (interaction.Request != null) { _reporter.Indent(); _reporter.ReportInfo(String.Format("with {0} {1}", interaction.Request.Method.ToString().ToUpper(), interaction.Request.Path)); } try { var interactionComparisonResult = ValidateInteraction(interaction); comparisonResult.AddChildResult(interactionComparisonResult); _reporter.Indent(); _reporter.ReportSummary(interactionComparisonResult); } finally { InvokeProviderStateTearDownIfApplicable(providerStateItem); InvokeTearDownIfApplicable(providerStates); } } _reporter.ResetIndentation(); _reporter.ReportFailureReasons(comparisonResult); _reporter.Flush(); if (comparisonResult.HasFailure) { throw new PactFailureException(String.Format("See test output or {0} for failure details.", !String.IsNullOrEmpty(_config.LoggerName) ? LogProvider.CurrentLogProvider.ResolveLogPath(_config.LoggerName) : "logs")); } } }
public void ReportSummary(ComparisonResult comparisonResult) { AddSummary(comparisonResult); }
public void ReportFailureReasons(ComparisonResult comparisonResult) { WriteFailureReasons(comparisonResult); }
private void WriteFailureReasons(ComparisonResult comparisonResult) { if (comparisonResult == null) { return; } if (!comparisonResult.HasFailure) { return; } AddReportLine(String.Empty, 0); AddReportLine("Failures:", 0); foreach (var failure in comparisonResult.Failures) { AddReportLine(String.Empty, 0); AddReportLine(String.Format("{0}) {1}", ++_failureCount, failure.Result), 0); } }
private Response HandleGetInteractionsVerificationRequest() { var registeredInteractions = _mockProviderRepository.TestScopedInteractions; var comparisonResult = new ComparisonResult(); //Check all registered interactions have been used once and only once if (registeredInteractions.Any()) { foreach (var registeredInteraction in registeredInteractions) { var interactionUsages = _mockProviderRepository.HandledRequests.Where(x => x.MatchedInteraction != null && x.MatchedInteraction == registeredInteraction).ToList(); if (interactionUsages == null || !interactionUsages.Any()) { comparisonResult.RecordFailure(new MissingInteractionComparisonFailure(registeredInteraction)); } else if (interactionUsages.Count() > 1) { comparisonResult.RecordFailure(new ErrorMessageComparisonFailure(String.Format("The interaction with description '{0}' and provider state '{1}', was used {2} time/s by the test.", registeredInteraction.Description, registeredInteraction.ProviderState, interactionUsages.Count()))); } } } //Have we seen any request that has not be registered by the test? if (_mockProviderRepository.HandledRequests != null && _mockProviderRepository.HandledRequests.Any(x => x.MatchedInteraction == null)) { foreach (var handledRequest in _mockProviderRepository.HandledRequests.Where(x => x.MatchedInteraction == null)) { comparisonResult.RecordFailure(new UnexpectedRequestComparisonFailure(handledRequest.ActualRequest)); } } //Have we seen any requests when no interactions were registered by the test? if (!registeredInteractions.Any() && _mockProviderRepository.HandledRequests != null && _mockProviderRepository.HandledRequests.Any()) { comparisonResult.RecordFailure(new ErrorMessageComparisonFailure("No interactions were registered, however the mock provider service was called.")); } if (!comparisonResult.HasFailure) { _log.Info("Verifying - interactions matched"); return GenerateResponse(HttpStatusCode.OK, "Interactions matched"); } _log.Error("Verifying - actual interactions do not match expected interactions"); if (comparisonResult.Failures.Any(x => x is MissingInteractionComparisonFailure)) { _log.Error("Missing requests: " + String.Join(", ", comparisonResult.Failures .Where(x => x is MissingInteractionComparisonFailure) .Cast<MissingInteractionComparisonFailure>() .Select(x => x.RequestDescription))); } if (comparisonResult.Failures.Any(x => x is UnexpectedRequestComparisonFailure)) { _log.Error("Unexpected requests: " + String.Join(", ", comparisonResult.Failures .Where(x => x is UnexpectedRequestComparisonFailure) .Cast<UnexpectedRequestComparisonFailure>() .Select(x => x.RequestDescription))); } foreach (var failureResult in comparisonResult.Failures.Where(failureResult => !(failureResult is MissingInteractionComparisonFailure) && !(failureResult is UnexpectedRequestComparisonFailure))) { _log.Error(failureResult.Result); } var failure = comparisonResult.Failures.First(); return GenerateResponse(HttpStatusCode.InternalServerError, failure.Result); }
public void ReportSummary_WithFailuresOnComparisonResult_CallsWriteErrorOnOutputterWithMessage() { const string comparisonMessage = "The thing I am testing"; var expectedMessage = String.Format("{0} (FAILED - 1)", comparisonMessage); var reporter = GetSubject(); var comparisonResult = new ComparisonResult(comparisonMessage); comparisonResult.RecordFailure(new ErrorMessageComparisonFailure("It failed")); reporter.ReportSummary(comparisonResult); _mockOutputter.Received(1).WriteError(expectedMessage, Arg.Any<int>()); }
public void Validate_WhenAFailureOccurs_ThrowsPactFailureException() { var pact = new ProviderServicePactFile { Consumer = new Pacticipant { Name = "My client" }, Provider = new Pacticipant { Name = "My Provider" }, Interactions = new List<ProviderServiceInteraction> { new ProviderServiceInteraction { Description = "My interaction" }, } }; var comparisonResult = new ComparisonResult(); comparisonResult.RecordFailure(new ErrorMessageComparisonFailure("It failed")); var validator = GetSubject(); _mockResponseComparer .Compare(Arg.Any<ProviderServiceResponse>(), Arg.Any<ProviderServiceResponse>()) .Returns(comparisonResult); Assert.Throws<PactFailureException>(() => validator.Validate(pact, null)); }
public void Flush_WithReportedFailureReasonThatContainsFailuresOnTheComparisonResult_CallsOutputterWithMessage() { const string comparisonMessage = "The thing I am testing"; const string comparisonFailureMessage1 = "It failed 1"; const string comparisonFailureMessage2 = "It failed 2"; var reporter = GetSubject(); var comparisonResult = new ComparisonResult(comparisonMessage); comparisonResult.RecordFailure(new ErrorMessageComparisonFailure(comparisonFailureMessage1)); comparisonResult.RecordFailure(new ErrorMessageComparisonFailure(comparisonFailureMessage2)); reporter.ReportFailureReasons(comparisonResult); reporter.Flush(); _reportOutputter.Received(1).Write(Arg.Is<string>(x => x.Contains("1) " + comparisonFailureMessage1) && x.Contains("2) " + comparisonFailureMessage2))); }
public void ReportSummary_WithNoFailuresOnComparisonResult_CallsWriteSuccessOnOutputterWithMessage() { const string comparisonMessage = "The thing I am testing"; var reporter = GetSubject(); var comparisonResult = new ComparisonResult(comparisonMessage); reporter.ReportSummary(comparisonResult); _mockOutputter.Received(1).WriteSuccess(comparisonMessage, Arg.Any<int>()); }
public void Flush_WithWithAReportedSummaryThatContainsNoFailuresOnTheComparisonResult_CallsOutputterWithMessage() { const string comparisonMessage = "The thing I am testing"; var reporter = GetSubject(); var comparisonResult = new ComparisonResult(comparisonMessage); reporter.ReportSummary(comparisonResult); reporter.Flush(); _reportOutputter.Received(1).Write(comparisonMessage); }
public void Flush_WithReportedFailureReasonThatContainsNoFailuresOnTheComparisonResult_DoesNotCallTheOutputter() { const string comparisonMessage = "The thing I am testing"; var reporter = GetSubject(); var comparisonResult = new ComparisonResult(comparisonMessage); reporter.ReportFailureReasons(comparisonResult); reporter.Flush(); _reportOutputter.DidNotReceive().Write(Arg.Any<string>()); }