public void FileError(TestError error)
{
lock (sync)
{
nestedCallback.FileError(error);
}
}
public void FileError(TestContext context, TestError error)
{
lock (sync)
{
nestedCallback.FileError(context, error);
}
}
/// <inheritdoc cref="IDisposable.Dispose" />
public void Dispose()
{
TestStream?.Dispose();
TestIn?.Dispose();
TestOut?.Dispose();
TestError?.Dispose();
}
private void BuildTestContexts(
TestOptions options,
IEnumerable <PathInfo> scriptPaths,
ParallelOptions parallelOptions,
CancellationTokenSource cancellationSource,
int resultCount,
ConcurrentBag <TestContext> testContexts,
ITestMethodRunnerCallback callback,
TestCaseSummary overallSummary)
{
Parallel.ForEach(scriptPaths, parallelOptions, testFile =>
{
ChutzpahTracer.TraceInformation("Building test context for {0}", testFile.FullPath);
try
{
if (cancellationSource.IsCancellationRequested)
{
return;
}
TestContext testContext;
resultCount++;
if (testContextBuilder.TryBuildContext(testFile, options, out testContext))
{
testContexts.Add(testContext);
}
else
{
ChutzpahTracer.TraceWarning("Unable to build test context for {0}", testFile.FullPath);
}
// Limit the number of files we can scan to attempt to build a context for
// This is important in the case of folder scanning where many JS files may not be
// test files.
if (resultCount >= options.FileSearchLimit)
{
ChutzpahTracer.TraceError("File search limit hit!!!");
cancellationSource.Cancel();
}
}
catch (Exception e)
{
var error = new TestError
{
InputTestFile = testFile.FullPath,
Message = e.ToString()
};
overallSummary.Errors.Add(error);
callback.FileError(error);
ChutzpahTracer.TraceError(e, "Error during building test context for {0}", testFile.FullPath);
}
finally
{
ChutzpahTracer.TraceInformation("Finished building test context for {0}", testFile.FullPath);
}
});
}
private static void HandleTestProcessExitCode(int exitCode, string inputTestFile, IList <TestError> errors, ITestMethodRunnerCallback callback)
{
string errorMessage = null;
switch ((TestProcessExitCode)exitCode)
{
case TestProcessExitCode.AllPassed:
case TestProcessExitCode.SomeFailed:
return;
case TestProcessExitCode.Timeout:
errorMessage = "Timeout occurred when executing test file";
break;
default:
errorMessage = "Unknown error occurred when executing test file. Received exit code of " + exitCode;
break;
}
if (!string.IsNullOrEmpty(errorMessage))
{
var error = new TestError
{
InputTestFile = inputTestFile,
Message = errorMessage
};
errors.Add(error);
callback.FileError(error);
ChutzpahTracer.TraceError("Headless browser returned with an error: {0}", errorMessage);
}
}
public Task <ApiResponse <Unit> > Handle(HattemSessionMock session, ErrorNotification notification)
{
var error = new TestError();
return(ApiResponse
.Error(error)
.AsTask());
}
public void ToFailureWhen_WhenConditionIsTrueWePropagateIError_ReturnFailureWithPropagatedError()
{
var err = new TestError("heheszki", "", "", 0);
var result = "leszek walesa"
.ToFailureWhen(x => !string.IsNullOrWhiteSpace(x), err);
(result is Failure <string>).ShouldBeTrue();
(result as Failure <string>).Error.ShouldBe(err);
}
public void on_failure_for_specific_error_called_on_same_failure()
{
var expectedError = new TestError();
var result = Failure(expectedError);
var actualError = default(TestError);
result.OnFailure<TestError>(error => actualError = error);
actualError.Should().Be(expectedError);
}
public void GetResponseFromResult_UnhandledError_ThrowsUnhandledErrorResultException()
{
// Arrange
var controllerBase = new TestController();
var error = new TestError();
var result = new Result(error);
// Act => Assert
Assert.Throws <UnhandledErrorResultException>(() => controllerBase.GetResponseFromResult(result));
}
public void ResultT_Sync_wizHandle_例外が発生した場合は例外ハンドラが実行されそこで返されたエラーが返る()
{
var ex = new Exception();
var er = new TestError();
Result.Try((Func <Result <string> >)(() => throw ex), e =>
{
e.Should().Be(ex);
return(er);
}).Should().BeError().And.Match(e => e == er);
}
public void the_error_is_passed_into_the_error_action()
{
var error = new TestError();
var result = AddResult <object> .WasError(error);
result
.Match(
success: value => Assert.True(false),
error: errs => Assert.True(errs.Contains(error))
);
}
public override void FileError(TestError error)
{
ClearCounter();
Console.ForegroundColor = ConsoleColor.Red;
var errorMessage = GetFileErrorMessage(error);
Console.WriteLine(errorMessage);
Console.ResetColor();
}
public async Task NoReturn_Async_wizHandle_例外が発生した場合は例外ハンドラが実行されそこで返されたエラーが返る()
{
var ex = new Exception();
var er = new TestError();
var res = await Result.Try((Func <Task>)(() => throw ex), e =>
{
e.Should().Be(ex);
return(er);
});
res.Should().BeError().And.Match(e => e == er);
}
public void can_create_an_error_result_from_single_errors()
{
var error = new TestError();
var result = AddResult <object> .WasError(error);
result
.Match(
success: value => Assert.False(true),
error: errors => Assert.True(true)
);
}
public void ExpiredAndInValidCreditCardNumberTest()
{
Validation <TestError, string> notEmpty(string str) =>
!string.IsNullOrEmpty(str)
? Success <TestError, string>(str)
: Fail <TestError, string>(TestError.New("must not be empty"));
Validation <TestError, string> isDigit(string str) =>
str.ForAll(Char.IsDigit)
? Success <TestError, string>(str)
: Fail <TestError, string>(TestError.New("failed isDigit"));
Func <string, Validation <TestError, string> > maxStrLength(int maxlenght) => str =>
str.Length < maxlenght
? Success <TestError, string>(str)
: Fail <TestError, string>(TestError.New("failed maxlenght"));
var res = maxStrLength(3)("aaa").Bind(s => isDigit(s));
res.Match(
Succ: _ =>
{
Console.WriteLine("no errors");
},
Fail: errors =>
{
errors.ToList().ForEach(error => Console.WriteLine(error));
});
var bothFailed = maxStrLength(3)("aadda") | isDigit("1e12");
bothFailed.Match(
Succ: _ =>
{
Console.WriteLine("no errors");
},
Fail: errors =>
{
errors.ToList().ForEach(error => Console.WriteLine(error));
});
var errorList = bothFailed.FailToArray();
var toEither = bothFailed.ToEither();
}
public async void async_then_will_return_a_task_if_it_is_an_error()
{
var error = new TestError();
var result = AddResult <object> .WasError(error);
var taskResult = await result
.ThenAsync(v => Task.FromResult(new object()));
taskResult
.Match(
success: value => Assert.False(true),
error: errors => Assert.True(errors.Contains(error))
);
}
public override void FileError(TestContext context, TestError error)
{
ClearCounter();
Console.ForegroundColor = ConsoleColor.Red;
var errorMessage = GetFileErrorMessage(error);
if (failOnError)
{
Console.Error.Write(errorMessage);
}
else
{
Console.Write(errorMessage);
}
Console.ResetColor();
}
public void ErrorWhenFetchingInstallations2()
{
var testError = new TestError("Error when fetching installations");
HomeModel.FetchVmItems
.Apply(i => throw new Exception("Should not call this function"))
.Apply(l => testError)
(new Location(50.062006, 19.940984)).Match(
error => Assert.Fail("Should match Right"),
tuple =>
{
var(errors, measurementVmItems) = tuple;
Assert.AreEqual(testError, errors.First());
Assert.AreEqual(1, errors.Count);
Assert.IsEmpty(measurementVmItems);
}
);
}
public async Task CanDeserializeCamelCaseError()
{
using (var server = LocalWebServer.ListenInBackground(new XUnitMockLogger(_logger)))
{
var uri = server.ListeningUri;
server
.WithNextResponse(new MockHttpResponseMessage(HttpStatusCode.NotFound).WithContent(TestError.SerializedDefault1).WithPrivateCacheHeader().WithDefaultExpiration());
var ex = await Should.ThrowAsync <HttpErrorException <TestError> >(
CreateBuilder()
.WithLink(uri)
.WithErrorType <TestError>()
.SendAsync());
ex.ShouldNotBeNull();
ex.Error.ShouldBe(TestError.Default1());
}
}
public void ErrorWhenFetchingOneOfMeasurements2()
{
TestItems(out var installation1, out var installation2, out var measurements1, out _,
out _, out _);
var testError = new TestError("Fetching not successful...");
Either <Error, (Measurements, Installation)> ErrorIfSecond(Installation i)
{
if (i.id == installation2.id)
{
return(testError);
}
else
{
return(measurements1, installation1);
}
}
var installations = new List <Installation> {
installation1, installation2
};
HomeModel.FetchVmItems
.Apply(ErrorIfSecond)
.Apply(location => installations)
(new Location(59.062006, 29.940984))
.Match(
error => Assert.Fail("Should match Right"),
tuple =>
{
var(errors, measurementVmItems) = tuple;
Assert.AreEqual(1, errors.Count);
Assert.AreEqual(testError, errors.First());
Assert.AreEqual(1, measurementVmItems.Count,
"Should not return installation if it's measurements are not fetched.");
Assert.AreEqual(installation1, measurementVmItems.First().Installation,
"Should return installation whose measurements were fetched successfully");
Assert.AreEqual(measurements1, measurementVmItems.First().Measurements,
"Should contain measurements returned by measurementById function");
});
}
public void Will_fire_error_event()
{
var reader = new TestableTestCaseStreamReader();
var json = JsonStreamEvents.FileStartEventJson + JsonStreamEvents.TestStartEventJson + JsonStreamEvents.ErrorEventJson;
var context = reader.BuildContext("file");
var stream = new StreamReader(new MemoryStream(Encoding.UTF8.GetBytes(json)));
var processStream = new ProcessStream(new Mock <IProcessWrapper>().Object, stream);
var callback = new Mock <ITestMethodRunnerCallback>();
TestError result = null;
callback.Setup(x => x.FileError(It.IsAny <TestError>())).Callback <TestError>(t => result = t);
reader.ClassUnderTest.Read(processStream, new TestOptions(), context, callback.Object, false);
Assert.NotNull(result);
Assert.Equal("file", result.InputTestFile);
Assert.Equal("error", result.Message);
Assert.Equal("errorFile", result.Stack[0].File);
Assert.Equal("errorFunc", result.Stack[0].Function);
Assert.Equal("22", result.Stack[0].Line);
}
public void Will_fire_error_event()
{
var reader = new TestableTestCaseStreamReader();
var json = @"#_#Error#_# {""type"":""Error"",""Error"":{""message"":""uhoh"", ""stack"":[{""file"":""errorFile"",""function"":""errorFunc"",""line"":22}]}}";
var context = new TestContext {
InputTestFile = "file"
};
var stream = new StreamReader(new MemoryStream(Encoding.UTF8.GetBytes(json)));
var processStream = new ProcessStream(new Mock <IProcessWrapper>().Object, stream);
var callback = new Mock <ITestMethodRunnerCallback>();
TestError result = null;
callback.Setup(x => x.FileError(It.IsAny <TestError>())).Callback <TestError>(t => result = t);
reader.ClassUnderTest.Read(processStream, new TestOptions(), context, callback.Object, false);
Assert.NotNull(result);
Assert.Equal("file", result.InputTestFile);
Assert.Equal("uhoh", result.Message);
Assert.Equal("errorFile", result.Stack[0].File);
Assert.Equal("errorFunc", result.Stack[0].Function);
Assert.Equal("22", result.Stack[0].Line);
}
public static string FormatFileErrorMessage(TestError error)
{
var stack = "";
foreach (var item in error.Stack)
{
if (!string.IsNullOrEmpty(item.Function))
{
stack += "at " + item.Function + " ";
}
if (!string.IsNullOrEmpty(item.File))
{
stack += "in " + item.File;
}
if (!string.IsNullOrEmpty(item.Line))
{
stack += string.Format(" (line {0})", item.Line);
}
stack += "\n";
}
return(string.Format("Error: {0}\n{1}While Running:{2}\n", error.Message, stack, error.InputTestFile));
}
private void ExecuteTestContexts(
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback,
ConcurrentBag <TestContext> testContexts,
ParallelOptions parallelOptions,
string headlessBrowserPath,
ConcurrentQueue <TestFileSummary> testFileSummaries,
TestCaseSummary overallSummary)
{
Parallel.ForEach(
testContexts,
parallelOptions,
testContext =>
{
ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.InputTestFile, testExecutionMode);
try
{
testHarnessBuilder.CreateTestHarness(testContext, options);
if (options.OpenInBrowser)
{
ChutzpahTracer.TraceInformation(
"Launching test harness '{0}' for file '{1}' in a browser",
testContext.TestHarnessPath,
testContext.InputTestFile);
process.LaunchFileInBrowser(testContext.TestHarnessPath);
}
else
{
ChutzpahTracer.TraceInformation(
"Invoking headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.InputTestFile);
var testSummary = InvokeTestRunner(
headlessBrowserPath,
options,
testContext,
testExecutionMode,
callback);
ChutzpahTracer.TraceInformation(
"Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors",
testContext.TestHarnessPath,
testContext.InputTestFile,
testSummary.PassedCount,
testSummary.FailedCount,
testSummary.Errors.Count);
ChutzpahTracer.TraceInformation(
"Finished running headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.InputTestFile);
testFileSummaries.Enqueue(testSummary);
}
}
catch (Exception e)
{
var error = new TestError
{
InputTestFile = testContext.InputTestFile,
Message = e.ToString()
};
overallSummary.Errors.Add(error);
callback.FileError(error);
ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.InputTestFile);
}
finally
{
ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.InputTestFile, testExecutionMode);
}
});
// Clean up test context
foreach (var testContext in testContexts)
{
// Don't clean up context if in debug mode
if (!m_debugEnabled && !options.OpenInBrowser)
{
try
{
ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.InputTestFile);
testContextBuilder.CleanupContext(testContext);
}
catch (Exception e)
{
ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.InputTestFile);
}
}
}
}
private void ExecuteTestContexts(
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback,
ConcurrentBag <TestContext> testContexts,
ParallelOptions parallelOptions,
string headlessBrowserPath,
ConcurrentQueue <TestFileSummary> testFileSummaries,
TestCaseSummary overallSummary)
{
Parallel.ForEach(
testContexts,
parallelOptions,
testContext =>
{
ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
try
{
try
{
testHarnessBuilder.CreateTestHarness(testContext, options);
}
catch (IOException)
{
// Mark this creation failed so we do not try to clean it up later
// This is to work around a bug in TestExplorer that runs chutzpah in parallel on
// the same files
// TODO(mmanela): Re-evalute if this is needed once they fix that bug
testContext.TestHarnessCreationFailed = true;
ChutzpahTracer.TraceWarning("Marking test harness creation failed for harness {0} and test file {1}", testContext.TestHarnessPath, testContext.FirstInputTestFile);
throw;
}
if (options.TestLaunchMode == TestLaunchMode.FullBrowser)
{
ChutzpahTracer.TraceInformation(
"Launching test harness '{0}' for file '{1}' in a browser",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
// Allow override from command line.
var browserArgs = testContext.TestFileSettings.BrowserArguments;
if (!string.IsNullOrWhiteSpace(options.BrowserArgs))
{
var path = BrowserPathHelper.GetBrowserPath(options.BrowserName);
browserArgs = new Dictionary <string, string>
{
{ Path.GetFileNameWithoutExtension(path), options.BrowserArgs }
};
}
process.LaunchFileInBrowser(testContext.TestHarnessPath, options.BrowserName, browserArgs);
}
else if (options.TestLaunchMode == TestLaunchMode.HeadlessBrowser)
{
ChutzpahTracer.TraceInformation(
"Invoking headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
var testSummaries = InvokeTestRunner(
headlessBrowserPath,
options,
testContext,
testExecutionMode,
callback);
foreach (var testSummary in testSummaries)
{
ChutzpahTracer.TraceInformation(
"Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors",
testContext.TestHarnessPath,
testSummary.Path,
testSummary.PassedCount,
testSummary.FailedCount,
testSummary.Errors.Count);
ChutzpahTracer.TraceInformation(
"Finished running headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testSummary.Path);
testFileSummaries.Enqueue(testSummary);
}
}
else if (options.TestLaunchMode == TestLaunchMode.Custom)
{
if (options.CustomTestLauncher == null)
{
throw new ArgumentNullException("TestOptions.CustomTestLauncher");
}
ChutzpahTracer.TraceInformation(
"Launching custom test on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
options.CustomTestLauncher.LaunchTest(testContext);
}
else
{
Debug.Assert(false);
}
}
catch (Exception e)
{
var error = new TestError
{
InputTestFile = testContext.InputTestFiles.FirstOrDefault(),
Message = e.ToString()
};
overallSummary.Errors.Add(error);
callback.FileError(error);
ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.FirstInputTestFile);
}
finally
{
ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
}
});
// Clean up test context
foreach (var testContext in testContexts)
{
// Don't clean up context if in debug mode
if (!m_debugEnabled &&
!testContext.TestHarnessCreationFailed &&
options.TestLaunchMode != TestLaunchMode.FullBrowser &&
options.TestLaunchMode != TestLaunchMode.Custom)
{
try
{
ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.FirstInputTestFile);
testContextBuilder.CleanupContext(testContext);
}
catch (Exception e)
{
ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.FirstInputTestFile);
}
}
}
}
public override void FileError(TestError error)
{
var errorMessage = GetFileErrorMessage(error);
Console.Write(errorMessage);
}
public virtual void FileError(TestError error)
{
}
protected virtual string GetFileErrorMessage(TestError error)
{
return(FormatFileErrorMessage(error));
}
public override void Visit (TestError node)
{
writer.WriteLine ("{0}) {1}\n{2}\n", ++id, GetName (), node.Error);
}
public override void FileError(TestContext context, TestError error)
{
testPane.OutputString(GetFileErrorMessage(error));
}
public ITestResult[] Test()
{
if (Ruleset != null)
{
List<ITestResult> results = new List<ITestResult>();
int testsToRun = 0;
foreach (IValidationRule rule in Ruleset.Rules)
{
if (rule.Tests != null)
testsToRun += rule.Tests.Length;
}
OnTestStart(testsToRun);
bool validatorsVerified = false;
foreach (IValidationRule rule in Ruleset.Rules)
{
if (rule.Tests != null)
{
foreach (ITest test in rule.Tests)
{
Debug.WriteLineWithTimestamp("Loading validators for test...");
// Load the ruleset validator using the test data.
RulesetValidator validator = new RulesetValidator(ResourceManager, Ruleset, test.iCalendarText, !validatorsVerified);
validatorsVerified = true;
Debug.WriteLineWithTimestamp("Done.");
// Validate the calendar!
IValidationResultCollection validationResults = validator.Validate();
TestResult result = new TestResult(ResourceManager, rule.Name, test, false);
Debug.WriteLineWithTimestamp("Interpreting test results...");
// Interpret the results...
if (test.Type == TestType.Fail)
{
if (BoolUtil.IsTrue(validationResults.Passed))
{
// The validation passed, but the test expected it to fail.
TestError error = new TestError(ResourceManager, "failExpectedError", rule.Name, validationResults);
error.Message = string.Format(error.Message, test.ExpectedError);
result.Error = error;
}
else
{
IValidationResultInfo[] details = validationResults.Details;
if (details.Length == 1 && !string.Equals(details[0].Name, test.ExpectedError))
{
// Validation failed (as expected), however, it failed with the incorrect error.
TestError error = new TestError(ResourceManager, "failWithIncorrectError", rule.Name, validationResults);
error.Message = string.Format(error.Message, details[0].Name, test.ExpectedError);
result.Error = error;
}
else if (details.Length > 1)
{
// Validation failed (as expected), however, it failed with more than one error.
TestError error = new TestError(ResourceManager, "failWithMoreThanOneError", rule.Name, validationResults);
error.Message = string.Format(error.Message, test.ExpectedError);
result.Error = error;
}
else
{
// The test passed, meaning the result was exactly as expected.
result.Passed = true;
}
}
}
else
{
if (BoolUtil.IsTrue(validationResults.Passed))
{
// The test passed, meaning the result was exactly as expected.
result.Passed = true;
}
else
{
// The validation was expected to succeed, but one or more errors occurred.
result.Passed = false;
result.Error = new TestError(ResourceManager, "passExpectedError", rule.Name, validationResults);
}
}
Debug.WriteLineWithTimestamp("Done.");
// Notify via event the result of the test.
if (result.Passed == null)
OnTestNotRun(result);
else if (BoolUtil.IsTrue(result.Passed))
OnTestPassed(result);
else
OnTestFailed(result);
results.Add(result);
// If we encounter a fatal error, then we cannot continue processing
// other errors. In other words, even though a single error was caused,
// the fact that it was fatal may have side effects and cause errors
// in almost every validator (i.e. a calendar with parsing errors).
if (validationResults.IsFatal)
break;
}
}
}
return results.ToArray();
}
// FIXME: else throw exception?
else return new ITestResult[0];
}
public static string FormatFileErrorMessage(TestError error)
{
var stack = error.GetFormattedStackTrace();
return(string.Format("Error: {0}\n{1}While Running:{2}\n", error.Message, stack, error.InputTestFile));
}