public void RunTestsForMultipleTestShouldSendMultipleResults()
{
var testCase = this.GetTestCase(typeof(DummyTestClass), "PassingTest");
var failingTestCase = this.GetTestCase(typeof(DummyTestClass), "FailingTest");
TestCase[] tests = new[] { testCase, failingTestCase };
this.TestExecutionManager.RunTests(tests, this.runContext, this.frameworkHandle, this.cancellationToken);
List <string> expectedTestCaseStartList = new List <string>()
{
"PassingTest", "FailingTest"
};
List <string> expectedTestCaseEndList = new List <string>()
{
"PassingTest:Passed", "FailingTest:Failed"
};
List <string> expectedResultList = new List <string>()
{
"PassingTest Passed", "FailingTest Failed\r\n Message: (null)"
};
CollectionAssert.AreEqual(expectedTestCaseStartList, this.frameworkHandle.TestCaseStartList);
CollectionAssert.AreEqual(expectedTestCaseEndList, this.frameworkHandle.TestCaseEndList);
Assert.AreEqual("PassingTest Passed", this.frameworkHandle.ResultsList[0]);
StringAssert.Contains(
this.frameworkHandle.ResultsList[1],
"FailingTest Failed\r\n Message: Assert.Fail failed. \r\n StackTrace:\r\n at Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution.TestExecutionManagerTests.DummyTestClass.FailingTest()");
}
public void ExecuteForClassInitializeThrowingExceptionShouldReturnUnitTestResultWithFailedOutcome()
{
// Arrange.
var tai = new TestAssemblyInfo(typeof(DummyTestClass).Assembly);
var constructorInfo = typeof(DummyTestClass).GetConstructors().Single();
var classAttribute = new UTF.TestClassAttribute();
var testContextProperty = typeof(DummyTestClass).GetProperty("TestContext");
var tci = new TestClassInfo(
type: typeof(DummyTestClass),
constructor: constructorInfo,
testContextProperty: testContextProperty,
classAttribute: classAttribute,
parent: tai)
{
ClassInitializeMethod = typeof(TestMethodRunnerTests).GetMethod(
"InitMethodThrowingException",
BindingFlags.Static | BindingFlags.NonPublic)
};
var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, tci, this.testMethodOptions, () => { throw new Exception("DummyException"); });
var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false);
// Act.
var results = testMethodRunner.Execute();
// Assert.
Assert.AreEqual(AdapterTestOutcome.Failed, results[0].Outcome);
StringAssert.Contains(results[0].ErrorMessage, "System.ArgumentException: Value does not fall within the expected range.");
}
public void AreEqualShouldFailWhenNotEqualDecimalWithMessage()
{
var ex = ActionUtility.PerformActionAndReturnException(() => TestFrameworkV2.Assert.AreEqual(0.1M, 0.2M, "A Message"));
Assert.IsNotNull(ex);
StringAssert.Contains(ex.Message, "A Message");
}
public void AreEqualShouldFailWhenFloatDoubleWithMessage()
{
var ex = ActionUtility.PerformActionAndReturnException(() => TestFrameworkV2.Assert.AreEqual(100E-2, 200E-2, "A Message"));
Assert.IsNotNull(ex);
StringAssert.Contains(ex.Message, "A Message");
}
public void ThrowsExceptionWithLamdaExpressionsShouldThrowAssertionOnNoException()
{
var ex = ActionUtility.PerformActionAndReturnException(() => TestFrameworkV2.Assert.ThrowsException <ArgumentException>(() => { }));
Assert.IsNotNull(ex);
Assert.AreEqual(typeof(TestFrameworkV2.AssertFailedException), ex.GetType());
StringAssert.Contains(ex.Message, "Assert.ThrowsException failed. No exception thrown. ArgumentException exception was expected.");
}
public void IsValidDeploymentItemShouldReportWarningIfDeploymentOutputDirectoryIsNull()
{
string warning;
Assert.IsFalse(this.deploymentItemUtility.IsValidDeploymentItem(this.defaultDeploymentItemPath, null, out warning));
StringAssert.Contains(Resource.DeploymentItemOutputDirectoryCannotBeNull, warning);
}
public void IsValidDeploymentItemShouldReportWarningIfSourcePathIsEmpty()
{
string warning;
Assert.IsFalse(this.deploymentItemUtility.IsValidDeploymentItem(string.Empty, this.defaultDeploymentItemOutputDirectory, out warning));
StringAssert.Contains(Resource.DeploymentItemPathCannotBeNullOrEmpty, warning);
}
public void IsFalseNullableBooleansShouldFailWithNull()
{
bool?nullBool = null;
var ex = ActionUtility.PerformActionAndReturnException(() => TestFrameworkV2.Assert.IsFalse(nullBool));
Assert.IsNotNull(ex);
StringAssert.Contains(ex.Message, "Assert.IsFalse failed");
}
public void WriteWithMessageShouldWriteToStringWriterForNullCharacters()
{
var stringWriter = new ThreadSafeStringWriter(null, "test");
this.testContextImplementation = new TestContextImplementation(this.testMethod.Object, stringWriter, this.properties);
this.testContextImplementation.Write("1 Testing \0 write \0");
StringAssert.Contains(stringWriter.ToString(), "1 Testing \\0 write \\0");
}
public void WriteShouldWriteToStringWriter()
{
var stringWriter = new ThreadSafeStringWriter(null, "test");
this.testContextImplementation = new TestContextImplementation(this.testMethod.Object, stringWriter, this.properties);
this.testContextImplementation.Write("{0} Testing write", 1);
StringAssert.Contains(stringWriter.ToString(), "1 Testing write");
}
public void WriteWithMessageShouldWriteToStringWriter()
{
var stringWriter = new StringWriter();
this.testContextImplementation = new TestContextImplementation(this.testMethod.Object, stringWriter, this.properties);
this.testContextImplementation.Write("1 Testing write");
StringAssert.Contains(stringWriter.ToString(), "1 Testing write");
}
public void WriteShouldWriteToStringWriterForNullCharacters()
{
var stringWriter = new StringWriter();
this.testContextImplementation = new TestContextImplementation(this.testMethod.Object, stringWriter, this.properties);
this.testContextImplementation.Write("{0} Testing \0 write \0", 1);
StringAssert.Contains(stringWriter.ToString(), "1 Testing \\0 write \\0");
}
public void GetDiagnosticMessagesShouldReturnMessagesFromWriteLine()
{
this.testContextImplementation = new TestContextImplementation(this.testMethod.Object, new StringWriter(), this.properties);
this.testContextImplementation.WriteLine("1 Testing write");
this.testContextImplementation.WriteLine("2 Its a happy day");
StringAssert.Contains(this.testContextImplementation.GetDiagnosticMessages(), "1 Testing write");
StringAssert.Contains(this.testContextImplementation.GetDiagnosticMessages(), "2 Its a happy day");
}
public void AddResultFileShouldThrowIfFileNameIsEmpty()
{
this.testContextImplementation = new TestContextImplementation(this.testMethod.Object, new System.IO.StringWriter(), this.properties);
var exception =
ActionUtility.PerformActionAndReturnException(() => this.testContextImplementation.AddResultFile(string.Empty));
Assert.AreEqual(typeof(ArgumentException), exception.GetType());
StringAssert.Contains(exception.Message, Resource.Common_CannotBeNullOrEmpty);
}
public void RunTestMethodForTestThrowingExceptionShouldReturnUnitTestResultWithFailedOutcome()
{
var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => { throw new Exception("Dummy Exception"); });
var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false);
var results = testMethodRunner.RunTestMethod();
Assert.AreEqual(AdapterTestOutcome.Failed, results[0].Outcome);
StringAssert.Contains(results[0].ErrorMessage, "Exception thrown while executing test");
}
public void IsValidDeploymentItemShouldReportWarningIfDeploymentOutputDirectoryIsRooted()
{
string warning;
Assert.IsFalse(this.deploymentItemUtility.IsValidDeploymentItem(this.defaultDeploymentItemPath, "C:\\temp", out warning));
StringAssert.Contains(
string.Format(
Resource.DeploymentItemOutputDirectoryMustBeRelative,
"C:\\temp"),
warning);
}
public void IsValidDeploymentItemShouldReportWarningIfOutputDirectoryHasInvalidCharacters()
{
string warning;
Assert.IsFalse(this.deploymentItemUtility.IsValidDeploymentItem(this.defaultDeploymentItemPath, "<>", out warning));
StringAssert.Contains(
string.Format(
Resource.DeploymentItemContainsInvalidCharacters,
this.defaultDeploymentItemPath,
"<>"),
warning);
}
public void RunClassInitializeShouldThrowTestFailedExceptionWithInconclusiveOnAssertInconclusive()
{
DummyTestClass.ClassInitializeMethodBody = (tc) => UTF.Assert.Inconclusive("Test Inconclusive");
this.testClassInfo.ClassInitializeMethod = typeof(DummyTestClass).GetMethod("ClassInitializeMethod");
var exception = ActionUtility.PerformActionAndReturnException(() => this.testClassInfo.RunClassInitialize(this.testContext)) as TestFailedException;
Assert.IsNotNull(exception);
Assert.AreEqual(UnitTestOutcome.Inconclusive, exception.Outcome);
StringAssert.Contains(exception.Message, "Test Inconclusive");
StringAssert.StartsWith(
exception.StackTraceInformation.ErrorStackTrace,
" at Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution.TestClassInfoTests.<>c.<RunClassInitializeShouldThrowTestFailedExceptionWithInconclusiveOnAssertInconclusive>");
}
public void RunClassCleanupShouldReturnAssertInconclusiveExceptionDetails()
{
// Arrange
DummyTestClass.ClassCleanupMethodBody = () => UTF.Assert.Inconclusive("Test Inconclusive");
this.testClassInfo.ClassCleanupMethod = typeof(DummyTestClass).GetMethod(nameof(DummyTestClass.ClassCleanupMethod));
// Act
var classCleanup = this.testClassInfo.RunClassCleanup();
// Assert
StringAssert.StartsWith(classCleanup, "Class Cleanup method DummyTestClass.ClassCleanupMethod failed.");
StringAssert.Contains(classCleanup, "Error Message: Assert.Inconclusive failed. Test Inconclusive.");
StringAssert.Contains(classCleanup, $"{typeof(TestClassInfoTests).FullName}.<>c.<{nameof(this.RunClassCleanupShouldReturnAssertInconclusiveExceptionDetails)}>");
}
public void RunClassCleanupShouldReturnExceptionDetailsOfNonAssertExceptions()
{
// Arrange
DummyTestClass.ClassCleanupMethodBody = () => { throw new ArgumentException("Argument Exception"); };
this.testClassInfo.ClassCleanupMethod = typeof(DummyTestClass).GetMethod(nameof(DummyTestClass.ClassCleanupMethod));
// Act
var classCleanup = this.testClassInfo.RunClassCleanup();
// Assert
StringAssert.StartsWith(classCleanup, "Class Cleanup method DummyTestClass.ClassCleanupMethod failed.");
StringAssert.Contains(classCleanup, "Error Message: System.ArgumentException: Argument Exception. Stack Trace:");
StringAssert.Contains(classCleanup, $"{typeof(TestClassInfoTests).FullName}.<>c.<{nameof(this.RunClassCleanupShouldReturnExceptionDetailsOfNonAssertExceptions)}>");
}
public void GetSettingsShouldThrowWhenParallelizeHasInvalidElements()
{
string runSettingxml =
@"<RunSettings>
<MSTestV2>
<Parallelize>
<Hola>Hi</Hola>
</Parallelize>
</MSTestV2>
</RunSettings>";
var exception = ActionUtility.PerformActionAndReturnException(() => MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias));
Assert.IsNotNull(exception);
Assert.AreEqual(typeof(AdapterSettingsException).FullName, exception.GetType().FullName);
StringAssert.Contains(exception.Message, "Invalid settings 'Parallelize'. Unexpected XmlElement: 'Hola'.");
}
public void GetSettingsShouldThrowIfParallelizationScopeIsNotValid()
{
string runSettingxml =
@"<RunSettings>
<MSTestV2>
<Parallelize>
<Scope>JustParallelizeWillYou</Scope>
</Parallelize>
</MSTestV2>
</RunSettings>";
var exception = ActionUtility.PerformActionAndReturnException(() => MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias));
Assert.IsNotNull(exception);
Assert.AreEqual(typeof(AdapterSettingsException).FullName, exception.GetType().FullName);
StringAssert.Contains(exception.Message, "Invalid value 'JustParallelizeWillYou' specified for 'Scope'. Supported scopes are ClassLevel, MethodLevel.");
}
public void GetSettingsShouldThrowIfParallelizationWorkersIsNegative()
{
string runSettingxml =
@"<RunSettings>
<MSTestV2>
<Parallelize>
<Workers>-1</Workers>
</Parallelize>
</MSTestV2>
</RunSettings>";
var exception = ActionUtility.PerformActionAndReturnException(() => MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias));
Assert.IsNotNull(exception);
Assert.AreEqual(typeof(AdapterSettingsException).FullName, exception.GetType().FullName);
StringAssert.Contains(exception.Message, "Invalid value '-1' specified for 'Workers'. The value should be a non-negative integer.");
}
public void ThrowsExceptionAsyncShouldThrowAssertionOnNoException()
{
Task t = TestFrameworkV2.Assert.ThrowsExceptionAsync <ArgumentException>(
async() =>
{
await Task.Delay(5);
});
var ex = ActionUtility.PerformActionAndReturnException(() => t.Wait());
Assert.IsNotNull(ex);
var innerException = ex.InnerException;
Assert.IsNotNull(innerException);
Assert.AreEqual(typeof(TestFrameworkV2.AssertFailedException), innerException.GetType());
StringAssert.Contains(innerException.Message, "Assert.ThrowsException failed. No exception thrown. ArgumentException exception was expected.");
}
public void RunBaseClassCleanupWithNoDerivedClassCleanupShouldReturnExceptionDetailsOfNonAssertExceptions()
{
// Arrange
DummyBaseTestClass.ClassCleanupMethodBody = () => { throw new ArgumentException("Argument Exception"); };
var baseClassCleanupMethod = typeof(DummyBaseTestClass).GetMethod(nameof(DummyBaseTestClass.CleanupClassMethod));
this.testClassInfo.ClassCleanupMethod = null;
this.testClassInfo.BaseClassInitAndCleanupMethods.Enqueue(Tuple.Create((MethodInfo)null, baseClassCleanupMethod));
this.testClassInfo.BaseClassCleanupMethodsStack.Push(baseClassCleanupMethod);
// Act
var classCleanup = this.testClassInfo.RunClassCleanup();
// Assert
StringAssert.StartsWith(classCleanup, "Class Cleanup method DummyBaseTestClass.CleanupClassMethod failed.");
StringAssert.Contains(classCleanup, "Error Message: System.ArgumentException: Argument Exception. Stack Trace:");
StringAssert.Contains(classCleanup, $"{typeof(TestClassInfoTests).FullName}.<>c.<{nameof(this.RunBaseClassCleanupWithNoDerivedClassCleanupShouldReturnExceptionDetailsOfNonAssertExceptions)}>");
}
public void ThrowsExceptionAsyncWithMessageShouldThrowAssertionOnWrongException()
{
Task t = TestFrameworkV2.Assert.ThrowsExceptionAsync <ArgumentException>(
async() =>
{
await Task.Delay(5);
throw new FormatException();
},
"Happily ever after.");
var ex = ActionUtility.PerformActionAndReturnException(() => t.Wait());
Assert.IsNotNull(ex);
var innerException = ex.InnerException;
Assert.IsNotNull(innerException);
Assert.AreEqual(typeof(TestFrameworkV2.AssertFailedException), innerException.GetType());
StringAssert.Contains(innerException.Message, "Assert.ThrowsException failed. Threw exception FormatException, but exception ArgumentException was expected. Happily ever after.");
}
public void ThrowsExceptionAsyncWithMessageAndParamsShouldThrowAssertionOnNoException()
{
Task t = TestFrameworkV2.Assert.ThrowsExceptionAsync <ArgumentException>(
async() =>
{
await Task.Delay(5);
},
"The world is not on fire {0}.{1}-{2}.",
"ta",
"da",
123);
var ex = ActionUtility.PerformActionAndReturnException(() => t.Wait());
Assert.IsNotNull(ex);
var innerException = ex.InnerException;
Assert.IsNotNull(innerException);
Assert.AreEqual(typeof(TestFrameworkV2.AssertFailedException), innerException.GetType());
StringAssert.Contains(innerException.Message, "Assert.ThrowsException failed. No exception thrown. ArgumentException exception was expected. The world is not on fire ta.da-123.");
}
public void ExecuteShouldFillInLogsIfAssemblyInitializeThrows()
{
StringWriter writer = new StringWriter(new StringBuilder());
DummyTestClass.AssemblyInitializeMethodBody = (UTFExtension.TestContext tc) =>
{
writer.Write("Hills");
tc.WriteLine("Valleys");
throw new ArgumentException();
};
this.testClassInfo.Parent.AssemblyInitializeMethod = typeof(DummyTestClass).GetMethod("DummyAssemblyInit");
var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult()
{
Outcome = UTF.UnitTestOutcome.Passed
});
var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, true);
this.testablePlatformServiceProvider.MockTraceListener.Setup(tl => tl.GetWriter()).Returns(writer);
var results = testMethodRunner.Execute();
Assert.AreEqual("Hills", results[0].DebugTrace);
StringAssert.Contains(results[0].TestContextMessages, "Valleys");
}