public virtual void MemoryLeakTests_ExitCodeTest_OnlyexitCodeTestResultAndNoWarnings()
{
string exitCodeTestName = "MemoryLeakTest";
MockOptions.Setup(o => o.ExitCodeTestCase).Returns(exitCodeTestName);
MockOptions.Setup(o => o.AdditionalTestExecutionParam).Returns("-is_run_by_gta");
var testCases = new GoogleTestDiscoverer(MockLogger.Object, TestEnvironment.Options,
new ProcessExecutorFactory(), new DefaultDiaResolverFactory()).GetTestsFromExecutable(TestResources.LeakCheckTests_DebugX86);
var testCase = testCases.Single(tc => tc.DisplayName.StartsWith(exitCodeTestName));
var executor = new TestExecutor(TestEnvironment.Logger, TestEnvironment.Options, MockDebuggerAttacher.Object);
executor.RunTests(testCase.Yield().Select(tc => tc.ToVsTestCase()), MockRunContext.Object, MockFrameworkHandle.Object);
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <VsTestResult>(result =>
result.TestCase.FullyQualifiedName.StartsWith("MemoryLeakTest") &&
result.Outcome == VsTestOutcome.Passed
)),
Times.Once);
MockFrameworkHandle.Verify(h => h.RecordResult(It.IsAny <VsTestResult>()), Times.Once);
MockLogger.Verify(l => l.DebugWarning(It.Is <string>(msg1 => msg1.Contains("main method") && msg1.Contains("exit code"))), Times.Never);
MockLogger.Verify(l => l.DebugWarning(It.Is <string>(msg => msg.Contains("test cases seem to not have been run"))), Times.Never);
}
protected override void CheckMockInvocations(int nrOfPassedTests, int nrOfFailedTests, int nrOfUnexecutedTests, int nrOfSkippedTests)
{
base.CheckMockInvocations(nrOfPassedTests, nrOfFailedTests, nrOfUnexecutedTests, nrOfSkippedTests);
if (nrOfPassedTests > 0)
{
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <TestResult>(tr => tr.Outcome == TestOutcome.Passed)),
Times.AtLeast(nrOfPassedTests));
MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny <TestCase>(), It.Is <TestOutcome>(to => to == TestOutcome.Passed)),
Times.AtLeast(nrOfPassedTests));
}
if (nrOfFailedTests > 0)
{
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <TestResult>(tr => tr.Outcome == TestOutcome.Failed)),
Times.AtLeast(nrOfFailedTests));
MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny <TestCase>(), It.Is <TestOutcome>(to => to == TestOutcome.Failed)),
Times.AtLeast(nrOfFailedTests));
}
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <TestResult>(tr => tr.Outcome == TestOutcome.Skipped)),
Times.AtMost(nrOfSkippedTests));
MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny <TestCase>(), It.Is <TestOutcome>(to => to == TestOutcome.Skipped)),
Times.AtMost(nrOfSkippedTests));
}
public void XmlReadSuccessOnlyTestCase()
{
var framework = new MockFrameworkHandle();
var runcontext = new MockRunContext();
var executor = new MockTestExecutor();
IList <string> xml_output = new List <string>()
{
@"<?xml version=""1.0"" encoding=""UTF-8""?>",
@"<Catch name=""CatchUnitTest.exe"">",
@" <Group name=""CatchUnitTest.exe"">",
@" <TestCase name=""First fixture"" tags=""[fixture]"" filename=""ReferenceCatchProject\fixture_test.cpp"" line=""13"">",
@" <OverallResult success=""true"" durationInSeconds=""0.00085""/>",
@" </TestCase>",
@" </Group>",
@" <OverallResults successes=""3"" failures=""3"" expectedFailures=""0""/>",
@"</Catch>"
};
IList <TestCase> tests = new List <TestCase>();
tests.Add(new TestCase("First fixture", new Uri(TestExecutor.ExecutorUriString), "ReferenceCatchProject")
{
CodeFilePath = "ReferenceCatchProject\fixture_test.cpp", LineNumber = 13
});
executor.MockComposeResults(xml_output, tests, framework);
Assert.AreEqual(1, framework.Results.Count);
Assert.AreEqual(TestOutcome.Passed, framework.Results[0].Outcome);
}
protected override void SetUpMockFrameworkHandle()
{
MockFrameworkHandle.Setup(
h => h.LaunchProcessWithDebuggerAttached(
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <IDictionary <string, string> >()))
.Returns((string filePath,
string workingDirectory,
string arguments,
IDictionary <string, string> environmentVariables) =>
{
var processStartInfo = new ProcessStartInfo(filePath, arguments)
{
WorkingDirectory = workingDirectory,
UseShellExecute = false,
CreateNoWindow = true
};
foreach (var kvp in environmentVariables)
{
processStartInfo.EnvironmentVariables[kvp.Key] = kvp.Value;
}
var process = new Process {
StartInfo = processStartInfo
};
if (!process.Start())
{
throw new Exception("WTF!");
}
return(process.Id);
});
}
public void WarningAndInfoMessage()
{
// Set up a fake testing context.
var framework = new MockFrameworkHandle();
// Execute all tests.
TestExecutor executor = new TestExecutor();
executor.RunTests(Common.ReferenceExeList, new MockRunContext(), framework);
// Map the tests by name.
Dictionary <string, TestResult> resultsByName = new Dictionary <string, TestResult>();
foreach (var result in framework.Results)
{
resultsByName[result.TestCase.FullyQualifiedName] = result;
}
TestResult testResult = resultsByName["Warn"];
Assert.AreEqual(TestOutcome.Failed, testResult.Outcome);
Assert.IsTrue(testResult.ErrorMessage.Contains("#1 - CHECK(false) with expansion: (false)"));
Assert.AreEqual(1, testResult.Messages.Count);
Assert.IsTrue(testResult.Messages[0].Text.Contains("WARN: This is a warning message"));
testResult = resultsByName["Info"];
Assert.AreEqual(TestOutcome.Failed, testResult.Outcome);
Assert.IsTrue(testResult.ErrorMessage.Contains("#1 - CHECK(false) with expansion: (false)"));
Assert.AreEqual(1, testResult.Messages.Count);
Assert.IsTrue(testResult.Messages[0].Text.Contains("INFO: This is a info message"));
}
public void FailureIsFailure()
{
// Set up a fake testing context.
var framework = new MockFrameworkHandle();
// Execute all tests.
TestExecutor executor = new TestExecutor();
executor.RunTests(Common.ReferenceExeList, new MockRunContext(), framework);
// Map the tests by name.
Dictionary <string, TestResult> resultsByName = new Dictionary <string, TestResult>();
foreach (var result in framework.Results)
{
resultsByName[result.TestCase.FullyQualifiedName] = result;
}
// Check that the failure failed and success succeeded.
TestResult failure = resultsByName["Has failure"];
Assert.AreEqual(TestOutcome.Failed, failure.Outcome);
Assert.IsTrue(failure.ErrorStackTrace.Contains("33")); // Failure line number.
TestResult success = resultsByName["With tags"];
Assert.AreEqual(TestOutcome.Passed, success.Outcome);
}
protected virtual void CheckMockInvocations(int nrOfPassedTests, int nrOfFailedTests, int nrOfUnexecutedTests, int nrOfNotFoundTests)
{
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult>(tr => tr.Outcome == Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome.None)),
Times.Exactly(nrOfUnexecutedTests));
MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase>(), It.Is <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome>(to => to == Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome.None)),
Times.Exactly(nrOfUnexecutedTests));
}
protected virtual void CheckMockInvocations(int nrOfPassedTests, int nrOfFailedTests, int nrOfUnexecutedTests, int nrOfSkippedTests)
{
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <VsTestResult>(tr => tr.Outcome == VsTestOutcome.None)),
Times.Exactly(nrOfUnexecutedTests));
MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny <VsTestCase>(), It.Is <VsTestOutcome>(to => to == VsTestOutcome.None)),
Times.Exactly(nrOfUnexecutedTests));
}
public void TestExecutableWithSpaces()
{
// Copy the reference exe to a path with a space.
string spaceDir = Directory.GetCurrentDirectory() + "\\C++ Space - Test";
string spaceExe = spaceDir + "\\Test Space.exe";
Directory.CreateDirectory(spaceDir);
File.Copy(Common.ReferenceExePath, spaceExe);
// Set up a fake testing context.
var framework = new MockFrameworkHandle();
// Execute all tests.
TestExecutor executor = new TestExecutor();
executor.RunTests(new List <string>()
{
spaceExe
}, new MockRunContext(), framework);
// Remove the copy.
File.Delete(spaceExe);
Directory.Delete(spaceDir, true);
// Make sure we got results for all.
Assert.AreEqual(Common.ReferenceTestCount, framework.Results.Count);
}
public void XmlReadTest()
{
var framework = new MockFrameworkHandle();
var runcontext = new MockRunContext();
var executor = new MockTestExecutor();
IList <string> xml_output = new List <string>()
{
@"<?xml version=""1.0"" encoding=""UTF-8""?>",
@"<Catch name=""CatchUnitTest.exe"">",
@" <Group name=""CatchUnitTest.exe"">",
@" <TestCase name=""Second fixture"" tags=""[fixture]"" filename=""ReferenceCatchProject\fixture_test.cpp"" line=""21"">",
@" <Expression success=""false"" type=""REQUIRE"" filename=""ReferenceCatchProject\fixture_test.cpp"" line=""25"">",
@" <Original>",
@" expected == actual",
@" </Original>",
@" <Expanded>",
@" true == false",
@" </Expanded>",
@" </Expression>",
@" <OverallResult success=""false"" durationInSeconds=""0.001865""/>",
@" </TestCase>",
@" <TestCase name=""Fail message test"" tags=""[fixture]"" filename=""ReferenceCatchProject\fixture_test.cpp"" line=""31"">",
@" <Failure filename=""ReferenceCatchProject\fixture_test.cpp"" line=""33"">",
@" Not implemented!",
@" </Failure>",
@" <OverallResult success=""false"" durationInSeconds=""0.001188""/>",
@" </TestCase>",
@" <TestCase name=""Fail message test in between"" tags=""[fixture]"" filename=""ReferenceCatchProject\fixture_test.cpp"" line=""36"">",
@" <Failure filename=""ReferenceCatchProject\fixture_test.cpp"" line=""40"">",
@" Not implemented!",
@" </Failure>",
@" <OverallResult success=""false"" durationInSeconds=""0.001295""/>",
@" </TestCase>",
@" <OverallResults successes=""3"" failures=""3"" expectedFailures=""0""/>",
@" </Group>",
@" <OverallResults successes=""3"" failures=""3"" expectedFailures=""0""/>",
@"</Catch>"
};
IList <TestCase> tests = new List <TestCase>();
tests.Add(new TestCase("Second fixture", new Uri(TestExecutor.ExecutorUriString), "ReferenceCatchProject")
{
CodeFilePath = "ReferenceCatchProject\fixture_test.cpp", LineNumber = 21
});
tests.Add(new TestCase("Fail message test", new Uri(TestExecutor.ExecutorUriString), "ReferenceCatchProject")
{
CodeFilePath = "ReferenceCatchProject\fixture_test.cpp", LineNumber = 31
});
tests.Add(new TestCase("Fail message test in between", new Uri(TestExecutor.ExecutorUriString), "ReferenceCatchProject")
{
CodeFilePath = "ReferenceCatchProject\fixture_test.cpp", LineNumber = 36
});
executor.MockComposeResults(xml_output, tests, framework);
Assert.AreEqual(3, framework.Results.Count);
for (int i = 0; i < framework.Results.Count; ++i)
{
Assert.AreEqual(TestOutcome.Failed, framework.Results[i].Outcome);
}
}
public void HelperFileTests_AdditionalParamsAreNotProvided_TestFails()
{
RunHelperFileTestsExecutable();
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <TestResult>(tr =>
tr.DisplayName.Contains("HelperFileTests.TheTargetIsSet") &&
tr.Outcome == TestOutcome.Failed)), Times.Once);
}
public virtual void RunTests_WithoutPathExtension_ExecutionFails()
{
var executor = new TestExecutor(TestEnvironment.Logger, TestEnvironment.Options);
executor.RunTests(TestResources.PathExtensionTestsExe.Yield(), MockRunContext.Object, MockFrameworkHandle.Object);
MockFrameworkHandle.Verify(h => h.RecordResult(It.IsAny <VsTestResult>()), Times.Never);
MockLogger.Verify(l => l.LogError(It.IsAny <string>()), Times.Once);
}
public void HelperFileTests_AdditionalParamsAreProvided_TestSucceeds()
{
MockOptions.Setup(o => o.AdditionalTestExecutionParam).Returns("-TheTarget=$(TheTarget)");
RunHelperFileTestsExecutable();
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <TestResult>(tr =>
tr.DisplayName.Contains("HelperFileTests.TheTargetIsSet") &&
tr.Outcome == TestOutcome.Passed)), Times.Once);
}
public virtual void RunTests_TestDirectoryViaUserParams_IsPassedViaCommandLineArg()
{
TestCase testCase = TestDataCreator.GetTestCases("CommandArgs.TestDirectoryIsSet").First();
RunAndVerifySingleTest(testCase, VsTestOutcome.Failed);
MockFrameworkHandle.Reset();
MockOptions.Setup(o => o.AdditionalTestExecutionParam).Returns("-testdirectory=\"" + SettingsWrapper.TestDirPlaceholder + "\"");
RunAndVerifySingleTest(testCase, VsTestOutcome.Passed);
}
public virtual void RunTests_WorkingDir_IsSetCorrectly()
{
TestCase testCase = TestDataCreator.GetTestCases("WorkingDir.IsSolutionDirectory").First();
MockOptions.Setup(o => o.WorkingDir).Returns(SettingsWrapper.ExecutableDirPlaceholder);
RunAndVerifySingleTest(testCase, VsTestOutcome.Failed);
MockFrameworkHandle.Reset();
MockOptions.Setup(o => o.WorkingDir).Returns(SettingsWrapper.SolutionDirPlaceholder);
RunAndVerifySingleTest(testCase, VsTestOutcome.Passed);
}
public virtual void RunTests_WithPathExtension_ExecutionOk()
{
MockOptions.Setup(o => o.PathExtension).Returns(SettingsWrapper.ExecutableDirPlaceholder + @"\..\lib");
var executor = new TestExecutor(TestEnvironment.Logger, TestEnvironment.Options);
executor.RunTests(TestResources.PathExtensionTestsExe.Yield(), MockRunContext.Object, MockFrameworkHandle.Object);
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <VsTestResult>(tr => tr.Outcome == VsTestOutcome.Passed)), Times.Exactly(29));
MockLogger.Verify(l => l.LogError(It.IsAny <string>()), Times.Never);
}
private void RunAndVerifySingleTest(TestCase testCase, VsTestOutcome expectedOutcome)
{
TestExecutor executor = new TestExecutor(TestEnvironment.Logger, TestEnvironment.Options);
executor.RunTests(testCase.ToVsTestCase().Yield(), MockRunContext.Object, MockFrameworkHandle.Object);
foreach (VsTestOutcome outcome in Enum.GetValues(typeof(VsTestOutcome)))
{
MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny <VsTestCase>(), It.Is <VsTestOutcome>(to => to == outcome)),
Times.Exactly(outcome == expectedOutcome ? 1 : 0));
}
}
public void TestExecutesAllTests()
{
// Set up a fake testing context.
var framework = new MockFrameworkHandle();
// Execute all tests.
TestExecutor executor = new TestExecutor();
executor.RunTests(Common.ReferenceExeList, new MockRunContext(), framework);
// Make sure we got results for all.
Assert.AreEqual(Common.ReferenceTestCount, framework.Results.Count);
}
public void LaunchProcessWithDebuggerAttached_WithParameters_PassedInfoToFrameworkHandleCorrectly()
{
DebuggedProcessLauncher launcher = new DebuggedProcessLauncher(MockFrameworkHandle.Object);
launcher.LaunchProcessWithDebuggerAttached("theCommand", "theDir", "theParams", "C:\\test");
MockFrameworkHandle.Verify(fh => fh.LaunchProcessWithDebuggerAttached(
It.Is <string>(s => s == "theCommand"),
It.Is <string>(s => s == "theDir"),
It.Is <string>(s => s == "theParams"),
It.Is <IDictionary <string, string> >(d => d.ContainsKey("PATH") && d["PATH"].StartsWith("C:\\test;"))
), Times.Exactly(1));
}
public void SubResult()
{
// Set up a fake testing context.
var framework = new MockFrameworkHandle();
// Execute all tests.
TestExecutor executor = new TestExecutor();
executor.RunTests(Common.ReferenceExeList, new MockRunContext(), framework);
Assert.IsTrue(framework.Results.Count != 0);
var resultsOfFoo = framework.Results.Where(r => r.TestCase.DisplayName == "Foo");
Assert.AreEqual(5, resultsOfFoo.Count());
Assert.IsTrue(resultsOfFoo.All(r => r.Outcome == TestOutcome.Failed));
}
private void RunExitCodeTest(string testCaseName, VsTestOutcome outcome)
{
string exitCodeTestName = "ExitCode";
MockOptions.Setup(o => o.ExitCodeTestCase).Returns(exitCodeTestName);
TestCase testCase = TestDataCreator.GetTestCases(testCaseName).First();
TestExecutor executor = new TestExecutor(TestEnvironment.Logger, TestEnvironment.Options, MockDebuggerAttacher.Object);
executor.RunTests(testCase.ToVsTestCase().Yield(), MockRunContext.Object, MockFrameworkHandle.Object);
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <VsTestResult>(result =>
result.TestCase.FullyQualifiedName == testCaseName &&
result.Outcome == outcome
)),
Times.Once);
// ReSharper disable once PossibleNullReferenceException
string finalName = exitCodeTestName + "." + Path.GetFileName(testCase.Source).Replace(".", "_");
bool outputAvailable = MockOptions.Object.UseNewTestExecutionFramework ||
!MockRunContext.Object.IsBeingDebugged;
Func <VsTestResult, bool> errorMessagePredicate = outcome == VsTestOutcome.Failed
? result => result.ErrorMessage.Contains("Exit code: 1") &&
(!outputAvailable || result.ErrorMessage.Contains("The result code output")) &&
!result.ErrorMessage.Contains(StreamingStandardOutputTestResultParser.GtaExitCodeOutputBegin) &&
!result.ErrorMessage.Contains(StreamingStandardOutputTestResultParser.GtaExitCodeOutputEnd) &&
!result.ErrorMessage.Contains("Some more output")
: (Func <VsTestResult, bool>)(result => string.IsNullOrEmpty(result.ErrorMessage) || result.ErrorMessage.Contains("The result code output"));
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <VsTestResult>(result =>
result.TestCase.FullyQualifiedName == finalName &&
result.Outcome == outcome &&
errorMessagePredicate(result)
)),
Times.Once);
MockFrameworkHandle.Verify(h => h.RecordResult(It.IsAny <VsTestResult>()), Times.Exactly(2));
if (!outputAvailable && outcome == VsTestOutcome.Failed)
{
MockLogger.Verify(l => l.LogWarning(It.Is <string>(msg => msg.Contains("Result code") &&
msg.Contains(SettingsWrapper.OptionUseNewTestExecutionFramework))), Times.Once);
}
MockLogger.Verify(l => l.DebugWarning(It.Is <string>(msg => msg.Contains("main method") && msg.Contains("exit code"))), Times.Never);
}
public void OneTestResult()
{
// Set up a fake testing context.
var framework = new MockFrameworkHandle();
// Execute all tests.
TestExecutor executor = new TestExecutor();
IList <TestCase> tests = new List <TestCase>();
tests.Add(new TestCase("Has forced failure", new Uri(TestExecutor.ExecutorUriString), "ReferenceCatchProject")
{
CodeFilePath = @"ReferenceCatchProject\Tests.cpp", LineNumber = 37
});
executor.RunTests(tests, new MockRunContext(), framework);
Assert.AreEqual(2, framework.Results.Count);
Assert.IsTrue(framework.Results.All(r => r.Outcome == TestOutcome.Failed));
}
public void ReportTestResults_FromVsTestConsole_StacktraceEndsWithoutNewline()
{
var errorStacktrace = "My stack trace";
var result = new TestResult(TestDataCreator.ToTestCase("MyTestCase"))
{
Outcome = TestOutcome.Failed,
ErrorStackTrace = errorStacktrace + Environment.NewLine,
ComputerName = "My Computer"
};
var reporter = new VsTestFrameworkReporter(MockFrameworkHandle.Object, false, MockLogger.Object);
reporter.ReportTestResults(result.Yield());
MockFrameworkHandle.Verify(h => h.RecordResult(
It.Is <VsTestResult>(tr => tr.ErrorStackTrace.Equals(errorStacktrace))),
Times.Exactly(1));
}
private void DoTestBeginOfErrorMessage(bool inVisualStudio, string beginOfErrorMessage)
{
var errorMessage = "My error message";
var result = new TestResult(TestDataCreator.ToTestCase("MyTestCase"))
{
Outcome = TestOutcome.Failed,
ErrorMessage = errorMessage,
ComputerName = "My Computer"
};
var reporter = new VsTestFrameworkReporter(MockFrameworkHandle.Object, inVisualStudio, MockLogger.Object);
reporter.ReportTestResults(result.Yield());
MockFrameworkHandle.Verify(h => h.RecordResult(
It.Is <VsTestResult>(tr => tr.ErrorMessage.Equals(beginOfErrorMessage + errorMessage))),
Times.Exactly(1));
}
override protected void CheckMockInvocations(int nrOfPassedTests, int nrOfFailedTests, int nrOfUnexecutedTests, int nrOfNotFoundTests)
{
base.CheckMockInvocations(nrOfPassedTests, nrOfFailedTests, nrOfUnexecutedTests, nrOfNotFoundTests);
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <TestResult>(tr => tr.Outcome == TestOutcome.Passed)),
Times.Exactly(nrOfPassedTests));
MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny <TestCase>(), It.Is <TestOutcome>(to => to == TestOutcome.Passed)),
Times.Exactly(nrOfPassedTests));
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <TestResult>(tr => tr.Outcome == TestOutcome.Failed)),
Times.Exactly(nrOfFailedTests));
MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny <TestCase>(), It.Is <TestOutcome>(to => to == TestOutcome.Failed)),
Times.Exactly(nrOfFailedTests));
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <TestResult>(tr => tr.Outcome == TestOutcome.Skipped)),
Times.Exactly(nrOfNotFoundTests));
MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny <TestCase>(), It.Is <TestOutcome>(to => to == TestOutcome.Skipped)),
Times.Exactly(nrOfNotFoundTests));
}
public virtual void RunTests_WithoutPathExtension_ExecutionFails()
{
string baseDir = TestDataCreator.PreparePathExtensionTest();
try
{
string targetExe = TestDataCreator.GetPathExtensionExecutable(baseDir);
var executor = new TestExecutor(TestEnvironment.Logger, TestEnvironment.Options);
executor.RunTests(targetExe.Yield(), MockRunContext.Object, MockFrameworkHandle.Object);
MockFrameworkHandle.Verify(h => h.RecordResult(It.IsAny <VsTestResult>()), Times.Never);
MockLogger.Verify(l => l.LogError(It.IsAny <string>()), Times.Once);
}
finally
{
Utils.DeleteDirectory(baseDir).Should().BeTrue();
}
}
public virtual void RunTests_WithPathExtension_ExecutionOk()
{
string baseDir = TestDataCreator.PreparePathExtensionTest();
try
{
string targetExe = TestDataCreator.GetPathExtensionExecutable(baseDir);
MockOptions.Setup(o => o.PathExtension).Returns(SettingsWrapper.ExecutableDirPlaceholder + @"\..\dll");
var executor = new TestExecutor(TestEnvironment.Logger, TestEnvironment.Options);
executor.RunTests(targetExe.Yield(), MockRunContext.Object, MockFrameworkHandle.Object);
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <VsTestResult>(tr => tr.Outcome == VsTestOutcome.Passed)), Times.Once);
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <VsTestResult>(tr => tr.Outcome == VsTestOutcome.Failed)), Times.Once);
MockLogger.Verify(l => l.LogError(It.IsAny <string>()), Times.Never);
}
finally
{
Utils.DeleteDirectory(baseDir).Should().BeTrue();
}
}
protected void RunMemoryLeakTest(string executable, string testCaseName, VsTestOutcome testOutcome, VsTestOutcome leakCheckOutcome, Func <string, bool> errorMessagePredicate)
{
string exitCodeTestName = "MemoryLeakTest";
MockOptions.Setup(o => o.ExitCodeTestCase).Returns(exitCodeTestName);
MockOptions.Setup(o => o.AdditionalTestExecutionParam).Returns("-is_run_by_gta");
var testCases = new GoogleTestDiscoverer(MockLogger.Object, TestEnvironment.Options,
new ProcessExecutorFactory(), new DefaultDiaResolverFactory()).GetTestsFromExecutable(executable);
var testCase = testCases.Single(tc => tc.DisplayName == testCaseName);
var executor = new TestExecutor(TestEnvironment.Logger, TestEnvironment.Options, MockDebuggerAttacher.Object);
executor.RunTests(testCase.Yield().Select(tc => tc.ToVsTestCase()), MockRunContext.Object, MockFrameworkHandle.Object);
var invocations = new List <IInvocation>();
foreach (var invocation in MockFrameworkHandle.Invocations)
{
invocations.Add(invocation);
}
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <VsTestResult>(result =>
result.TestCase.FullyQualifiedName == testCaseName &&
result.Outcome == testOutcome
)),
Times.Once);
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <VsTestResult>(result =>
result.TestCase.FullyQualifiedName.StartsWith("MemoryLeakTest") &&
result.Outcome == leakCheckOutcome &&
(result.ErrorMessage == null || !result.ErrorMessage.Contains(StreamingStandardOutputTestResultParser.GtaExitCodeOutputBegin)) &&
(result.ErrorMessage == null || !result.ErrorMessage.Contains(StreamingStandardOutputTestResultParser.GtaExitCodeOutputEnd)) &&
errorMessagePredicate(result.ErrorMessage)
)),
Times.Once);
MockLogger.Verify(l => l.DebugWarning(It.Is <string>(msg => msg.Contains("main method") && msg.Contains("exit code"))), Times.Never);
}
public void BrokenXmlWithSingleTest()
{
var framework = new MockFrameworkHandle();
var runcontext = new MockRunContext();
var executor = new MockTestExecutor();
IList <string> xml_output = new List <string>()
{
@"<?xml version=""1.0"" encoding=""UTF-8""?>",
@"<Catch name=""CatchUnitTest.exe"">",
@" <Group name=""CatchUnitTest.exe"">",
@" <TestCase name=""C++ assert"" filename=""ReferenceCatchProject\testrunnertest.cpp"" line=""45"">"
};
IList <TestCase> tests = new List <TestCase>();
tests.Add(new TestCase("C++ assert", new Uri(TestExecutor.ExecutorUriString), "ReferenceCatchProject")
{
CodeFilePath = "ReferenceCatchProject\testrunnertest.cpp", LineNumber = 45
});
executor.MockComposeResults(xml_output, tests, framework);
Assert.AreEqual(1, framework.Results.Count);
Assert.AreEqual(TestOutcome.None, framework.Results[0].Outcome);
}
public void XmlPassedTestWithInfo()
{
var framework = new MockFrameworkHandle();
var runcontext = new MockRunContext();
var executor = new MockTestExecutor();
IList <string> xml_output = new List <string>()
{
@"<?xml version=""1.0"" encoding=""UTF-8""?>",
@"<Catch name = ""ReferenceCatchProject.exe"" >",
@" <Group name=""ReferenceCatchProject.exe"">",
@" <TestCase name = ""First fixture"" tags=""[fixture]"" filename=""fixture_test.cpp"" line=""13"">",
@" <Expression success = ""true"" type=""CHECK"" filename=""fixture_test.cpp"" line=""17"">",
@" <Original>",
@" expected == actual",
@" </Original>",
@" <Expanded>",
@" true == true",
@" </Expanded>",
@" </Expression>",
@" <OverallResult success = ""true"" durationInSeconds=""0.143113""/>",
@" </TestCase>",
@" <OverallResults successes = ""1"" failures=""0"" expectedFailures=""0""/>",
@" </Group>",
@" <OverallResults successes = ""1"" failures=""0"" expectedFailures=""0""/>",
@"</Catch>"
};
IList <TestCase> tests = new List <TestCase>();
tests.Add(new TestCase("First fixture", new Uri(TestExecutor.ExecutorUriString), "ReferenceCatchProject")
{
CodeFilePath = @"ReferenceCatchProject\fixture_test.cpp", LineNumber = 13
});
executor.MockComposeResults(xml_output, tests, framework);
var results = framework.Results.ToArray();
Assert.AreEqual(1, framework.Results.Count);
Assert.AreEqual(TestOutcome.Passed, results[0].Outcome);
}