private TestStepState GetOrCreateTestStepStateByTest(MSTest test)
{
TestStepState testStepState = GetTestStepStateByTest(test);
if (testStepState == null)
{
TestStepState parentTestStepState = GetOrCreateTestStepStateByTest((MSTest)test.Parent);
Guid testId = test.Guid;
ITestCommand testCommand = GetTestCommandByTestId(testId);
TestStep testStep = new TestStep(test, parentTestStepState.TestContext.TestStep,
test.Name, test.CodeElement, true);
if (test.IsDataDriven)
{
testStep.IsTestCase = false;
}
ITestContext testContext = SafeStartCommandStep(parentTestStepState.TestContext, testCommand, testStep);
testStepState = new TestStepState(parentTestStepState, testContext);
testStepStatesByTest.Add(test, testStepState);
testsExecuted.Add(test);
}
return(testStepState);
}
private TestStepState GetOrCreateTestStepStateByTestResultId(object testResultId)
{
TestStepState testStepState = GetTestStepStateByTestResultId(testResultId);
if (testStepState != null)
{
return(testStepState);
}
object testResult = GetTestResult(testResultId);
object testObj = testResult.GetType().GetProperty("Test").GetValue(testResult, null);
object testIdObj = testObj.GetType().GetProperty("Id").GetValue(testObj, null);
Guid testId = (Guid)testIdObj.GetType().GetProperty("Id").GetValue(testIdObj, null);
ITestCommand testCommand = GetTestCommandByTestId(testId);
if (testCommand == null)
{
return(null);
}
testStepState = GetOrCreateTestStepStateByTest((MSTest)testCommand.Test);
testStepState.TestResultId = testResultId;
return(testStepState);
}
async private Task WriteFinalStepState(TestStepState stepState, ITestOutputHelper output)
{
const int NumberOfRetries = 5;
FileStream stepStateStream = null;
// Retry few times because the state file may be open temporarily by another thread or process.
for (int retries = 0; retries < NumberOfRetries; retries++)
{
try
{
stepStateStream = File.Open(_stateFilePath, FileMode.Open, FileAccess.ReadWrite, FileShare.Read);
break;
}
catch (IOException ex)
{
output.WriteLine("WriteFinalStepState exception {0} retry #{1}", ex.ToString(), retries);
if (retries >= (NumberOfRetries - 1))
{
throw;
}
}
}
using (stepStateStream)
{
stepStateStream.Seek(0, SeekOrigin.End);
StreamWriter writer = new StreamWriter(stepStateStream);
await writer.WriteAsync(Environment.NewLine + stepState.SerializeFinalState());
await writer.FlushAsync();
}
}
private async Task UncachedExecute(TestStepState stepState, ITestOutputHelper output)
{
using (FileTestOutputHelper stepLog = new FileTestOutputHelper(_logFilePath))
{
try
{
LogHeader(stepState, false, output);
MultiplexTestOutputHelper mux = new MultiplexTestOutputHelper(new IndentedTestOutputHelper(output), stepLog);
await DoWork(mux);
stepState = stepState.Complete();
}
catch (Exception e)
{
stepState = stepState.Fault(e.Message, e.StackTrace);
}
finally
{
LogFooter(stepState, output);
await WriteFinalStepState(stepState, output);
ThrowExceptionIfFaulted(stepState);
}
}
}
private void StartStep(Report report, TestStepData step)
{
TestData testData = GetTestData(report, step.TestId);
TestStepRun testStepRun = new TestStepRun(step);
testStepRun.StartTime = DateTime.Now;
TestStepState parentState;
if (step.ParentId != null)
{
parentState = GetTestStepState(step.ParentId);
parentState.TestStepRun.Children.Add(testStepRun);
}
else
{
parentState = null;
report.TestPackageRun.RootTestStepRun = testStepRun;
}
TestStepState state = new TestStepState(parentState, testData, testStepRun);
states.Add(step.Id, state);
eventDispatcher.NotifyTestStepStarted(
new TestStepStartedEventArgs(report, testData, testStepRun));
}
private void LogFooter(TestStepState stepState, ITestOutputHelper output)
{
output.WriteLine("}");
string elapsedTime = null;
if (stepState.RunState == TestStepRunState.InProgress)
{
output.WriteLine(FriendlyName + " Not Complete");
output.WriteLine(stepState.ErrorMessage);
}
else
{
elapsedTime = (stepState.CompleteTime.Value - stepState.StartTime).ToString("mm\\:ss\\.fff");
}
if (stepState.RunState == TestStepRunState.Complete)
{
output.WriteLine(FriendlyName + " Complete (" + elapsedTime + " elapsed)");
}
else if (stepState.RunState == TestStepRunState.Faulted)
{
output.WriteLine(FriendlyName + " Faulted (" + elapsedTime + " elapsed)");
output.WriteLine(stepState.ErrorMessage);
output.WriteLine(stepState.ErrorStackTrace);
}
output.WriteLine("");
output.WriteLine("");
}
private void ThrowExceptionIfFaulted(TestStepState cachedStepState)
{
if (cachedStepState.RunState == TestStepRunState.Faulted)
{
throw new TestStepException(FriendlyName, cachedStepState.ErrorMessage, cachedStepState.ErrorStackTrace);
}
}
private bool TryWriteInitialStepState(TestStepState state, ITestOutputHelper output)
{
// To ensure the file is atomically updated we write the contents to a temporary
// file, then move it to the final location
try
{
string tempPath = Path.GetTempFileName();
try
{
File.WriteAllText(tempPath, state.SerializeInitialState());
Directory.CreateDirectory(Path.GetDirectoryName(_stateFilePath));
File.Move(tempPath, _stateFilePath);
return(true);
}
finally
{
File.Delete(tempPath);
}
}
catch (IOException ex)
{
output.WriteLine("Exception writing state file {0} {1}", _stateFilePath, ex.ToString());
return(false);
}
}
public static bool TryParse(string text, out TestStepState parsedState)
{
parsedState = null;
try
{
// The XmlReader is not happy with two root nodes so we crudely split them.
int indexOfInitialStepStateElementEnd = text.IndexOf("</InitialStepState>");
if (indexOfInitialStepStateElementEnd == -1)
{
return(false);
}
int splitIndex = indexOfInitialStepStateElementEnd + "</InitialStepState>".Length;
string initialStepStateText = text.Substring(0, splitIndex);
string finalStepStateText = text.Substring(splitIndex);
XElement initialStepStateElement = XElement.Parse(initialStepStateText);
if (initialStepStateElement == null || initialStepStateElement.Name != "InitialStepState")
{
return(false);
}
XElement machineElement = initialStepStateElement.Element("Machine");
if (machineElement == null || string.IsNullOrWhiteSpace(machineElement.Value))
{
return(false);
}
string machine = machineElement.Value;
XElement processIDElement = initialStepStateElement.Element("ProcessID");
int processID;
if (processIDElement == null ||
!processIDElement.Value.StartsWith("0x"))
{
return(false);
}
string processIdNumberText = processIDElement.Value.Substring("0x".Length);
if (!int.TryParse(processIdNumberText, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out processID))
{
return(false);
}
string processName = null;
XElement processNameElement = initialStepStateElement.Element("ProcessName");
if (processNameElement != null)
{
processName = processNameElement.Value;
}
DateTimeOffset startTime;
XElement startTimeElement = initialStepStateElement.Element("StartTime");
if (startTimeElement == null || !DateTimeOffset.TryParse(startTimeElement.Value, out startTime))
{
return(false);
}
parsedState = new TestStepState(TestStepRunState.InProgress, machine, processID, processName, startTime, null, null, null);
TryParseFinalState(finalStepStateText, ref parsedState);
return(true);
}
catch (XmlException)
{
return(false);
}
}
private void LogHeader(TestStepState stepState, bool cached, ITestOutputHelper output)
{
string cachedText = cached ? " (CACHED)" : "";
output.WriteLine("[" + stepState.StartTime + "] " + FriendlyName + cachedText);
output.WriteLine("Process: " + stepState.ProcessName + "(ID: 0x" + stepState.ProcessID.ToString("x") + ") on " + stepState.Machine);
output.WriteLine("{");
}
public TestStepState(TestStepState parent, TestData testData, TestStepRun testStepRun)
{
Parent = parent;
TestData = testData;
TestStepRun = testStepRun;
LogWriter = new StructuredDocumentWriter();
testStepRun.TestLog = LogWriter.Document;
}
private void InitializeLookupTables()
{
foreach (ITestCommand testCommand in assemblyTestCommand.PreOrderTraversal)
{
testCommandsByTestId.Add(((MSTest)testCommand.Test).Guid, testCommand);
}
assemblyTestStepState = new TestStepState(null, assemblyTestContext);
testStepStatesByTest.Add(assemblyTestCommand.Test, assemblyTestStepState);
}
private static void TryParseFinalState(string text, ref TestStepState taskState)
{
// If there are errors reading the final state portion of the stream we need to treat it
// as if the stream had terminated at the end of the InitialTaskState node.
// This covers a small window of time when appending the FinalTaskState node is in progress.
//
if (string.IsNullOrWhiteSpace(text))
{
return;
}
try
{
XElement finalTaskStateElement = XElement.Parse(text);
if (finalTaskStateElement == null || finalTaskStateElement.Name != "FinalStepState")
{
return;
}
XElement runStateElement = finalTaskStateElement.Element("RunState");
TestStepRunState runState;
if (runStateElement == null || !Enum.TryParse <TestStepRunState>(runStateElement.Value, out runState))
{
return;
}
DateTimeOffset?completeTime = null;
XElement completeTimeElement = finalTaskStateElement.Element("CompleteTime");
if (completeTimeElement != null)
{
DateTimeOffset tempCompleteTime;
if (!DateTimeOffset.TryParse(completeTimeElement.Value, out tempCompleteTime))
{
return;
}
else
{
completeTime = tempCompleteTime;
}
}
XElement errorMessageElement = finalTaskStateElement.Element("ErrorMessage");
string errorMessage = null;
if (errorMessageElement != null)
{
errorMessage = errorMessageElement.Value;
}
XElement errorStackTraceElement = finalTaskStateElement.Element("ErrorStackTrace");
string errorStackTrace = null;
if (errorStackTraceElement != null)
{
errorStackTrace = errorStackTraceElement.Value;
}
taskState = taskState.WithFinalState(runState, completeTime, errorMessage, errorStackTrace);
}
catch (XmlException) { }
}
private void HandleTestStepLifecyclePhaseChangedMessage(TestStepLifecyclePhaseChangedMessage message)
{
reportBox.Write(report =>
{
ThrowIfDisposed();
string stepId = RedirectTestStepId(message.StepId);
TestStepState state = GetTestStepState(stepId);
eventDispatcher.NotifyTestStepLifecyclePhaseChanged(
new TestStepLifecyclePhaseChangedEventArgs(report, state.TestData, state.TestStepRun, message.LifecyclePhase));
});
}
private IEnumerable <TestStepState> GetOrCreateTestStepStatesFromTestResultEventArgs(EventArgs testResultEventArgs)
{
object[] testResultIds = (object[])testResultEventArgs.GetType().GetProperty("ResultIds").GetValue(testResultEventArgs, null);
foreach (object testResultId in testResultIds)
{
TestStepState testStepState = GetOrCreateTestStepStateByTestResultId(testResultId);
if (testStepState != null)
{
yield return(testStepState);
}
}
}
private void HandleTestStepMetadataAddedMessage(TestStepMetadataAddedMessage message)
{
reportBox.Write(report =>
{
ThrowIfDisposed();
string stepId = RedirectTestStepId(message.StepId);
TestStepState state = GetTestStepState(stepId);
state.TestStepRun.Step.Metadata.Add(message.MetadataKey, message.MetadataValue);
eventDispatcher.NotifyTestStepMetadataAdded(
new TestStepMetadataAddedEventArgs(report, state.TestData, state.TestStepRun, message.MetadataKey, message.MetadataValue));
});
}
private void HandleTestStepLogAttachMessage(TestStepLogAttachMessage message)
{
reportBox.Write(report =>
{
ThrowIfDisposed();
string stepId = RedirectTestStepId(message.StepId);
TestStepState state = GetTestStepState(stepId);
state.LogWriter.Attach(message.Attachment);
eventDispatcher.NotifyTestStepLogAttach(
new TestStepLogAttachEventArgs(report, state.TestData, state.TestStepRun, message.Attachment));
});
}
private static bool IsTestCaseAncestorOrSelf(TestStepState state)
{
do
{
if (state.TestStepRun.Step.IsTestCase)
{
return(true);
}
state = state.Parent;
}while (state != null);
return(false);
}
private void HandleTestStepLogStreamEndBlockMessage(TestStepLogStreamEndBlockMessage message)
{
reportBox.Write(report =>
{
ThrowIfDisposed();
string stepId = RedirectTestStepId(message.StepId);
TestStepState state = GetTestStepState(stepId);
state.LogWriter[message.StreamName].End();
eventDispatcher.NotifyTestStepLogStreamEndBlock(
new TestStepLogStreamEndBlockEventArgs(report, state.TestData, state.TestStepRun, message.StreamName));
});
}
private void FinishStep(Report report, string stepId, TestResult result)
{
TestStepState state = GetTestStepState(stepId);
state.TestStepRun.EndTime = DateTime.Now;
state.TestStepRun.Result = result;
PromoteToTestCaseIfStepAppearsToHaveBlockedChildrenFromRunning(state);
report.TestPackageRun.Statistics.MergeStepStatistics(state.TestStepRun);
state.LogWriter.Close();
eventDispatcher.NotifyTestStepFinished(
new TestStepFinishedEventArgs(report, state.TestData, state.TestStepRun));
}
private async Task <TestStepState> AcquireStepStateLock(ITestOutputHelper output)
{
TestStepState initialStepState = new TestStepState();
bool stepStateFileExists = false;
while (true)
{
TestStepState openedStepState = null;
stepStateFileExists = File.Exists(_stateFilePath);
if (!stepStateFileExists && TryWriteInitialStepState(initialStepState, output))
{
// this thread gets to do the work, persist the initial lock state
return(initialStepState);
}
if (stepStateFileExists && TryOpenExistingStepStateFile(out openedStepState, output))
{
if (!ShouldReuseCachedStepState(openedStepState))
{
try
{
File.Delete(_stateFilePath);
continue;
}
catch (IOException ex)
{
output.WriteLine("Exception deleting state file {0} {1}", _stateFilePath, ex.ToString());
}
}
else if (openedStepState.RunState != TestStepRunState.InProgress)
{
// we can reuse the work and it is finished - stop waiting and return it
return(openedStepState);
}
}
// If we get here we are either:
// a) Waiting for some other thread (potentially in another process) to complete the work
// b) Waiting for a hopefully transient IO issue to resolve so that we can determine whether or not the work has already been claimed
//
// If we wait for too long in either case we will eventually timeout.
ThrowExceptionForIncompleteWorkIfNeeded(initialStepState, openedStepState, stepStateFileExists, output);
await Task.Delay(TimeSpan.FromSeconds(1));
}
}
private void FinishTestCommand(ITestCommand testCommand)
{
FinishTestCommandChildren(testCommand);
MSTest test = (MSTest)testCommand.Test;
if (test.IsTestCase)
{
string ignoreReason = test.Metadata.GetValue(MetadataKeys.IgnoreReason);
if (!String.IsNullOrEmpty(ignoreReason))
{
TestStepState testStepState = GetOrCreateTestStepStateByTest(test);
testStepState.TestContext.LogWriter.Warnings.Write(string.Format("Test was ignored: {0}", ignoreReason));
testStepState.Outcome = TestOutcome.Ignored;
testStepState.Finish();
}
else
{
TestStepState testStepState = GetTestStepStateByTest(test);
if (testStepState == null)
{
if (!testsExecuted.Contains(test))
{
testStepState = GetOrCreateTestStepStateByTest(test);
testStepState.TestContext.LogWriter.Warnings.Write("No test results available!");
testStepState.Outcome = TestOutcome.Skipped;
testStepState.Finish();
}
}
else
{
testStepState.Finish();
}
}
}
else
{
TestStepState testStepState = GetTestStepStateByTest(test);
if (testStepState != null)
{
testStepState.Finish();
}
}
}
async public Task Execute(ITestOutputHelper output)
{
// if this step is in progress on another thread, wait for it
TestStepState stepState = await AcquireStepStateLock(output);
//if this thread wins the race we do the work on this thread, otherwise
//we log the winner's saved output
if (stepState.RunState != TestStepRunState.InProgress)
{
LogHeader(stepState, true, output);
LogPreviousResults(stepState, output);
LogFooter(stepState, output);
ThrowExceptionIfFaulted(stepState);
}
else
{
await UncachedExecute(stepState, output);
}
}
private void LogPreviousResults(TestStepState cachedTaskState, ITestOutputHelper output)
{
ITestOutputHelper indentedOutput = new IndentedTestOutputHelper(output);
try
{
string[] lines = File.ReadAllLines(_logFilePath);
foreach (string line in lines)
{
indentedOutput.WriteLine(line);
}
}
catch (IOException e)
{
string errorMessage = "Error accessing task log file: " + _logFilePath + Environment.NewLine +
e.GetType().FullName + ": " + e.Message;
indentedOutput.WriteLine(errorMessage);
}
}
private static string GetReuseStepStateReason(TestStepState openedStepState)
{
//This heuristic may need to change, in some cases it is probably too eager to
//reuse past results when we wanted to retest something.
if (openedStepState.RunState == TestStepRunState.Complete)
{
return("succesful steps are always reused");
}
else if (!IsPreviousMachineSame(openedStepState))
{
return("steps on run on other machines are always reused, regardless of success");
}
else if (IsPreviousProcessRunning(openedStepState))
{
return("steps run in currently executing processes are always reused, regardless of success");
}
else
{
return(null);
}
}
private bool TryOpenExistingStepStateFile(out TestStepState stepState, ITestOutputHelper output)
{
stepState = null;
try
{
if (!Directory.Exists(Path.GetDirectoryName(_stateFilePath)))
{
return(false);
}
bool result = TestStepState.TryParse(File.ReadAllText(_stateFilePath), out stepState);
if (!result)
{
output.WriteLine("TryParse failed on opening existing state file {0}", _stateFilePath);
}
return(result);
}
catch (IOException ex)
{
output.WriteLine("Exception opening existing state file {0} {1}", _stateFilePath, ex.ToString());
return(false);
}
}
private void FinishTestStep(TestStepState testStepState, object testResult)
{
Array innerResults = GetInnerResults(testResult);
if (innerResults != null)
{
for (int i = 0; i < innerResults.Length; i++)
{
object innerResult = innerResults.GetValue(i);
TestStep testStep = testStepState.TestContext.TestStep;
TestStep innerTestStep = new TestStep(testStep.Test, testStep,
testStep.Name, testStep.CodeElement, false);
innerTestStep.IsDynamic = true;
Array innerInnerResults = GetInnerResults(innerResults);
if (innerInnerResults != null && innerInnerResults.Length != 0)
innerTestStep.IsTestCase = false;
ITestContext innerTestContext = SafeStartChildStep(testStepState.TestContext, innerTestStep);
TestStepState innerTestStepState = new TestStepState(testStepState, innerTestContext);
FinishTestStep(innerTestStepState, innerResult);
}
}
Type testResultType = testResult.GetType();
string stdOut = (string)testResultType.GetProperty("StdOut").GetValue(testResult, null);
if (!string.IsNullOrEmpty(stdOut))
testStepState.TestContext.LogWriter.ConsoleOutput.Write(stdOut);
string stdErr = (string)testResultType.GetProperty("StdErr").GetValue(testResult, null);
if (!string.IsNullOrEmpty(stdErr))
testStepState.TestContext.LogWriter.ConsoleError.Write(stdErr);
string debugTrace = (string)testResultType.GetProperty("DebugTrace").GetValue(testResult, null);
if (!string.IsNullOrEmpty(debugTrace))
testStepState.TestContext.LogWriter.DebugTrace.Write(debugTrace);
string errorMessage = (string)testResultType.GetProperty("ErrorMessage").GetValue(testResult, null);
if (!string.IsNullOrEmpty(errorMessage))
testStepState.TestContext.LogWriter.Failures.Write(errorMessage);
string errorStackTrace = (string)testResultType.GetProperty("ErrorStackTrace").GetValue(testResult, null);
if (!string.IsNullOrEmpty(errorStackTrace))
{
if (!string.IsNullOrEmpty(errorMessage))
testStepState.TestContext.LogWriter.Failures.WriteLine();
testStepState.TestContext.LogWriter.Failures.Write(errorStackTrace);
}
string[] textMessages = (string[])testResultType.GetProperty("TextMessages").GetValue(testResult, null);
foreach (string textMessage in textMessages)
testStepState.TestContext.LogWriter.Warnings.WriteLine(textMessage);
string outcomeString = testResultType.GetProperty("Outcome").GetValue(testResult, null).ToString();
testStepState.Outcome = GetTestOutcome(outcomeString);
Array timerResults = (Array)testResultType.GetProperty("TimerResults").GetValue(testResult, null);
if (timerResults != null)
{
for (int i = 0; i < timerResults.Length; i++)
{
object timerResult = timerResults.GetValue(i);
TimeSpan duration =
(TimeSpan)timerResult.GetType().GetProperty("Duration").GetValue(timerResult, null);
testStepState.Duration += duration;
}
}
// Finish the test step unless it is the assembly test which we will finish later.
if (testStepState.ParentTestStepState != null)
testStepState.Finish();
if (testStepState.TestResultId != null)
testStepStatesByTestResultId.Remove(testStepState.TestResultId);
if (testStepState.TestContext.TestStep.IsPrimary)
testStepStatesByTest.Remove(testStepState.TestContext.TestStep.Test);
}
private static bool IsOpenedStateChangeable(TestStepState openedStepState)
{
return(openedStepState.RunState == TestStepRunState.InProgress &&
IsPreviousMachineSame(openedStepState) &&
IsPreviousProcessRunning(openedStepState));
}
private static bool IsPreviousProcessRunning(TestStepState openedStepState)
{
Debug.Assert(IsPreviousMachineSame(openedStepState));
return(Process.GetProcesses().Any(p => p.Id == openedStepState.ProcessID && p.ProcessName == openedStepState.ProcessName));
}
private static bool IsPreviousMachineSame(TestStepState openedStepState)
{
return(Environment.MachineName == openedStepState.Machine);
}
private static bool ShouldReuseCachedStepState(TestStepState openedStepState)
{
return(GetReuseStepStateReason(openedStepState) != null);
}
private TestStepState GetOrCreateTestStepStateByTest(MSTest test)
{
TestStepState testStepState = GetTestStepStateByTest(test);
if (testStepState == null)
{
TestStepState parentTestStepState = GetOrCreateTestStepStateByTest((MSTest) test.Parent);
Guid testId = test.Guid;
ITestCommand testCommand = GetTestCommandByTestId(testId);
TestStep testStep = new TestStep(test, parentTestStepState.TestContext.TestStep,
test.Name, test.CodeElement, true);
if (test.IsDataDriven)
testStep.IsTestCase = false;
ITestContext testContext = SafeStartCommandStep(parentTestStepState.TestContext, testCommand, testStep);
testStepState = new TestStepState(parentTestStepState, testContext);
testStepStatesByTest.Add(test, testStepState);
testsExecuted.Add(test);
}
return testStepState;
}
private void InitializeLookupTables()
{
foreach (ITestCommand testCommand in assemblyTestCommand.PreOrderTraversal)
testCommandsByTestId.Add(((MSTest)testCommand.Test).Guid, testCommand);
assemblyTestStepState = new TestStepState(null, assemblyTestContext);
testStepStatesByTest.Add(assemblyTestCommand.Test, assemblyTestStepState);
}
public TestStepState(TestStepState parent, TestData testData, TestStepRun testStepRun)
{
Parent = parent;
TestData = testData;
TestStepRun = testStepRun;
LogWriter = new StructuredDocumentWriter();
testStepRun.TestLog = LogWriter.Document;
}
private void StartStep(Report report, TestStepData step)
{
TestData testData = GetTestData(report, step.TestId);
TestStepRun testStepRun = new TestStepRun(step);
testStepRun.StartTime = DateTime.Now;
TestStepState parentState;
if (step.ParentId != null)
{
parentState = GetTestStepState(step.ParentId);
parentState.TestStepRun.Children.Add(testStepRun);
}
else
{
parentState = null;
report.TestPackageRun.RootTestStepRun = testStepRun;
}
TestStepState state = new TestStepState(parentState, testData, testStepRun);
states.Add(step.Id, state);
eventDispatcher.NotifyTestStepStarted(
new TestStepStartedEventArgs(report, testData, testStepRun));
}
private static bool IsTestCaseAncestorOrSelf(TestStepState state)
{
do
{
if (state.TestStepRun.Step.IsTestCase)
return true;
state = state.Parent;
}
while (state != null);
return false;
}
public TestStepState(TestStepState parentTestStepState, ITestContext testContext)
{
this.parentTestStepState = parentTestStepState;
this.testContext = testContext;
}
/// <summary>
/// In some situations, we may receive a report that a test step representing an
/// inner node of the test tree failed and therefore prevented other test cases
/// from running. When this happens, we automatically promote the test step to
/// behave as if it were a test case and report the failure.
/// </summary>
/// <remarks>
/// This is really a hack to make up for the fact that most of the information
/// presented to users is about test cases rather than test suites and other inner
/// nodes of the test tree. Because test cases can be constructed dynamically,
/// we have a bit of a problem counting and presenting them when inner nodes fail.
/// I hope someday we come up with a better solution to this issue with our test model.
/// Perhaps we could introduce a "blocked" status.
/// -- Jeff.
/// </remarks>
private static void PromoteToTestCaseIfStepAppearsToHaveBlockedChildrenFromRunning(TestStepState state)
{
if (state.TestStepRun.Result.Outcome.Status != TestStatus.Passed
&& state.TestStepRun.Children.Count == 0
&& ! IsTestCaseAncestorOrSelf(state))
{
state.TestStepRun.Step.IsTestCase = true;
}
}
public TestOutcome Execute(string testMetadataPath, string testResultsPath,
string runConfigPath, string searchPathRoot)
{
TextWriter writer = assemblyTestContext.LogWriter["MSTest Output"];
string executablePath = MSTestResolver.FindMSTestPathForVisualStudioVersion(runner.GetVisualStudioVersion());
if (executablePath == null)
{
assemblyTestContext.LogWriter.Failures.Write(Resources.MSTestController_MSTestExecutableNotFound);
return TestOutcome.Error;
}
string executableDir = Path.GetDirectoryName(executablePath);
string privateAssembliesDir = Path.Combine(executableDir, "PrivateAssemblies");
string publicAssembliesDir = Path.Combine(executableDir, "PublicAssemblies");
RuntimeAccessor.AssemblyLoader.AddHintDirectory(executableDir);
RuntimeAccessor.AssemblyLoader.AddHintDirectory(privateAssembliesDir);
RuntimeAccessor.AssemblyLoader.AddHintDirectory(publicAssembliesDir);
// Obtain an Executor.
Assembly commandLineAssembly = Assembly.Load(runner.GetCommandLineAssemblyName());
Type executorType = commandLineAssembly.GetType("Microsoft.VisualStudio.TestTools.CommandLine.Executor");
object executor = Activator.CreateInstance(executorType);
try
{
// Configure the Executor's Output to send output to the assembly log writer.
PropertyInfo outputProperty = executorType.GetProperty("Output",
BindingFlags.Public | BindingFlags.Static);
object output = outputProperty.GetValue(executor, null);
FieldInfo standardOutputField = output.GetType().GetField("m_standardOutput",
BindingFlags.NonPublic | BindingFlags.Instance);
standardOutputField.SetValue(output, new StreamWriterAdapter(writer));
// Register commands with the executor to set command-line arguments.
Type commandFactoryType =
commandLineAssembly.GetType("Microsoft.VisualStudio.TestTools.CommandLine.CommandFactory");
CreateAndAddCommand(executor, commandFactoryType, "/nologo", null);
CreateAndAddCommand(executor, commandFactoryType, "/noisolation", null);
CreateAndAddCommand(executor, commandFactoryType, "/testmetadata", testMetadataPath);
CreateAndAddCommand(executor, commandFactoryType, "/resultsfile", testResultsPath);
CreateAndAddCommand(executor, commandFactoryType, "/runconfig", runConfigPath);
CreateAndAddCommand(executor, commandFactoryType, "/searchpathroot", searchPathRoot);
CreateAndAddCommand(executor, commandFactoryType, "/testlist", SelectedTestListName);
// Get the TMI.
tmi = commandFactoryType.GetProperty("Tmi", BindingFlags.Public | BindingFlags.Static).GetValue(null, null);
// Add event handlers.
AddEventHandler(tmi, "TestRunStartedEvent", HandleTestRunStarted);
AddEventHandler(tmi, "TestRunFinishedEvent", HandleTestRunFinished);
AddEventHandler(tmi, "TestStartedEvent", HandleTestStarted);
AddEventHandler(tmi, "TestFinishedEvent", HandleTestFinished);
// Execute!
InitializeLookupTables();
bool success = (bool)executorType.GetMethod("Execute").Invoke(executor, null);
TestOutcome assemblyOutcome = assemblyTestStepState.Outcome;
if (!success)
assemblyOutcome = TestOutcome.Error;
return assemblyOutcome;
}
catch (Exception ex)
{
assemblyTestContext.LogWriter.Failures.WriteException(ex, "A fatal exception occurred while running MSTest tests.");
return TestOutcome.Error;
}
finally
{
// Release state.
assemblyTestStepState = null;
tmi = null;
testCommandsByTestId.Clear();
testStepStatesByTestResultId.Clear();
testStepStatesByTest.Clear();
testsExecuted.Clear();
// Dispose the Executor. (Also disposes the TMI behind the scenes.)
((IDisposable)executor).Dispose();
}
}
