internal HarnessExecutionResult Run(string[] args)
{
TimedTest test;
HarnessExecutionResult hResult = HarnessExecutionResult.Unavailable;
bool runTestsIndividually = false;
m_log = new XmlLog();
// Prelim: Set the env, parse the arguments, set the result paths and set the test list.
SetEnvironment();
if (args != null && args.Length > 0)
{
if (!ParseArguments(args, ref runTestsIndividually))
{
return HarnessExecutionResult.InvalidArguments;
}
}
SetResultPaths();
BaseTest[] testList = BuildTestList(runTestsIndividually);
// Create a new harness object and set the properties.
Harness harness = new Harness(IsDevEnvironment);
if (m_transport != null)
{
harness.Transport = m_transport;
}
else
{
// harness constructor assigns default transport
m_transport = harness.Transport;
}
if (m_device != null)
{
harness.Device = m_device;
}
else
{
// harness constructor assigns default device
m_device = harness.Device;
}
// Execute each of the solution files using Harness.
for (int i = 0; i < testList.Length; i++)
{
if (testList[i] == null)
{
continue;
}
if (this.Transport.ToLower().Contains("tcpip") &&
(testList[i].Name.ToLower().Contains("netinfotests.sln")))
{
continue;
}
hResult = HarnessExecutionResult.Unavailable;
int attempts = 0;
while ((hResult != HarnessExecutionResult.Success &&
hResult != HarnessExecutionResult.Abort) && attempts++ < 3)
{
test = new TimedTest(testList[i], harness, m_log);
// Kill any emulators running from previous runs.
TerminateRunningEmulators(test, testList[i]);
try
{
hResult = test.Execute();
if (hResult == HarnessExecutionResult.Unavailable)
{
Utils.WriteToEventLog("Harness returned an unavailable result after running the test: " +
testList[i].Name + ". No of tries so far = " + attempts);
string deviceStatus = DeviceStatus(hResult);
Utils.WriteToEventLog("Device status after unavailable from harness: " + deviceStatus);
}
// Test did not execute because the device was dead.
// If so, reset power to the device and re-run test.
if (hResult == HarnessExecutionResult.NoConnection)
{
Utils.WriteToEventLog("Harness returned an NoConnection result after running the test: " +
testList[i].Name + ". No of tries so far = " + attempts);
string deviceStatus = DeviceStatus(hResult);
Utils.WriteToEventLog("Device status after noconnection from harness: " + deviceStatus);
}
// Test did not succeed running in three attempts.
if (hResult == HarnessExecutionResult.TimeOut)
{
Utils.WriteToEventLog("Test: " + test.Test.Name + " failed.");
harness.MFTestResult = Harness.Result.Fail;
GetTestResultDetails(harness, testList[i]);
Console.WriteLine("Test Result: " + harness.MFTestResult);
test.SendMail();
break;
}
// Test did not succeed running in three attempts.
if (hResult != HarnessExecutionResult.Success && attempts >= 3)
{
Utils.WriteToEventLog("Test: " + test.Test.Name + " failed.");
harness.MFTestResult = Harness.Result.Fail;
GetTestResultDetails(harness, testList[i]);
Console.WriteLine("Test Result: " + harness.MFTestResult);
}
// Test succeeded with 3 attempts or an abort was sent by harness.
if ((hResult == HarnessExecutionResult.Success && attempts < 4) ||
(hResult == HarnessExecutionResult.Abort))
{
GetTestResultDetails(harness, testList[i]);
if (!string.IsNullOrEmpty(m_device))
{
string deviceStatus = DeviceStatus(hResult);
Utils.WriteToEventLog("Device status after running " + testList[i].Name
+ ": " + deviceStatus);
if (!IsProfilerRun)
{
m_log.AddDeviceStatusToLog("Device ping result after running "
+ testList[i].Name + ": " + deviceStatus);
}
if (string.Equals(deviceStatus.ToLower(), "noconnection"))
{
throw new ApplicationException("Device did not reboot correctly after " +
"running the test: " + testList[i].Name);
}
}
if (!IsProfilerRun)
{
Console.WriteLine("Test Result: " + harness.MFTestResult);
}
}
}
catch (Exception ex)
{
if (ex is FileNotFoundException)
{
if (!IsProfilerRun)
{
Console.WriteLine(ex.ToString());
try
{
m_log.AddCommentToLog(ex.ToString());
}
catch
{
}
Utils.WriteToEventLog(
string.Format("Exception in TestSystem.cs: {0}", ex.ToString()));
hResult = HarnessExecutionResult.Abort;
}
}
}
// Wait for a few seconds before starting the next test when running on devices.
if (!string.Equals(m_transport.ToLower(), "emulator"))
{
System.Threading.Thread.Sleep(5000);
}
}
}
// Update test results and logs location.
m_didAllTestsPass = ((tests.FailCount > 0) || (tests.PassCount == 0)) ? false : true;
UpdateLogFolder();
return hResult;
}
/// <summary>
/// Overloaded Constructor
/// </summary>
/// <param name="test">The name of the sln file.</param>
/// <param name="harness">The Harness instance.</param>
/// <param name="log">Path to log file. </param>
internal TimedTest(BaseTest test, Harness harness, XmlLog log)
{
this.m_test = test;
this.m_harness = harness;
this.m_log = log;
}
internal HarnessExecutionResult Run(BaseTest test, XmlLog log)
{
m_initialTime = new DateTime();
string file = string.Empty;
string csprojFilePath = string.Empty;
m_logFileSaved = false;
m_debugText = new StringBuilder();
Thread desktopThread = null;
NamedPipeServerStream pipeStream = null;
try
{
// Set the file paths.
file = BuildFilePath(test, ref m_xslPath);
Console.WriteLine("\nTest: " + test.Name);
string pathForFile = file.Substring(0, file.LastIndexOf(@"\"));
if (!pathForFile.EndsWith(@"\"))
{
pathForFile = string.Format(@"{0}\", pathForFile);
}
string slnFileName = test.Name.Substring(test.Name.LastIndexOf(@"\") + 1);
try
{
// Get reference list and build the test
ArrayList referenceList = GetProjectReferences(file, ref csprojFilePath);
if (m_currentAppType != TestType.DeviceDesktop)
{
BuildTest(csprojFilePath, m_currentAppType);
}
else
{
pipeStream = new
NamedPipeServerStream("MFHarnessPipe", PipeDirection.InOut, 1,
PipeTransmissionMode.Message, PipeOptions.WriteThrough);
}
// Set log file name.
SetLogFileName(test.Name);
test.LogFile = m_outputXmlFileName;
if (!(test is ProfilerTest))
{
m_log = log;
m_log.StartLog(m_outputXmlFileName, m_xslPath);
}
// Get build and exe path.
string buildPath = GetBuildPath(file);
test.ExeLocation = buildPath + m_assemblyName + ".exe";
// If this is a profiler test, run the test and return.
if (test is ProfilerTest)
{
return RunProfilerTest(file, test.ExeLocation, buildPath, referenceList);
}
string testName = m_logFileName.Replace(".xml", string.Empty);
StartCodeCoverage(testName);
m_startTime = DateTime.Now;
switch(m_currentAppType)
{
case TestType.Device:
try
{
RunDeviceTest(buildPath, test.ExeLocation, referenceList);
}
catch (Exception ex)
{
Close();
return HarnessResult(ex);
}
break;
case TestType.DeviceDesktop:
// Desktop apps under
// DevBox: %spoclient%\Test\Platform\Tests\Desktop\Applications\
// TestBox: %programfiles%\Microsoft .NET Micro Framework\<version>\Tests\Desktop\
desktopThread = StartDesktopApplication(pathForFile, ref csprojFilePath);
pipeStream.WaitForConnection();
// Get references and build the device test.
GetProjectReferences(file, ref csprojFilePath);
BuildTest(csprojFilePath, TestType.Device);
RunDeviceTest(buildPath, test.ExeLocation, referenceList);
break;
case TestType.Desktop:
RunDesktopTest(test.ExeLocation);
break;
}
m_endTime = DateTime.Now;
StopCodeCoverage(testName);
}
catch (Exception ex)
{
m_endTime = DateTime.Now;
if (null != m_log)
{
m_log.WriteElementString("Test_Exception", ex.Message + ex.StackTrace);
}
Utils.WriteToEventLog("Exception: " + ex.ToString());
Close();
if (ex is ApplicationException && ex.Message.ToLower().Contains("build failure"))
{
return HarnessExecutionResult.Abort;
}
else
{
return HarnessExecutionResult.Unavailable;
}
}
finally
{
if (m_currentAppType == TestType.DeviceDesktop)
{
StopDesktopApplication(pipeStream, desktopThread);
}
Close();
if (null != m_log)
{
switch(m_currentAppType)
{
case TestType.DeviceDesktop:
SaveLogFile(test.Location, TestType.DeviceDesktop);
break;
default:
SaveLogFile(test.Location);
break;
}
}
// Change test results back to unknown for the next test.
TestResults = Result.NotKnown;
}
if (!string.IsNullOrEmpty(file))
{
if (!string.Equals(m_transport.ToLower(), "emulator"))
{
m_log.SynchronizeLogTime(file, m_initialTime);
}
}
else
{
return HarnessExecutionResult.Unavailable;
}
}
catch (Exception ex)
{
if (ex is FileNotFoundException)
{
throw ex;
}
Utils.WriteToEventLog(string.Format("Exception in Harness: {0}", ex.ToString()));
return HarnessExecutionResult.Abort;
}
return HarnessExecutionResult.Success;
}
internal HarnessExecutionResult Run(BaseTest test, XmlLog log)
{
m_initialTime = new DateTime();
string file = string.Empty;
string csprojFilePath = string.Empty;
m_logFileSaved = false;
m_debugText = new StringBuilder();
Thread desktopThread = null;
NamedPipeServerStream pipeStream = null;
try
{
// Set the file paths.
file = BuildFilePath(test, ref m_xslPath);
Console.WriteLine("\nTest: " + test.Name);
string pathForFile = file.Substring(0, file.LastIndexOf(@"\"));
if (!pathForFile.EndsWith(@"\"))
{
pathForFile = string.Format(@"{0}\", pathForFile);
}
string slnFileName = test.Name.Substring(test.Name.LastIndexOf(@"\") + 1);
try
{
// Get reference list and build the test
ArrayList referenceList = GetProjectReferences(file, ref csprojFilePath);
if (m_currentAppType != TestType.DeviceDesktop)
{
BuildTest(csprojFilePath, m_currentAppType);
}
else
{
pipeStream = new
NamedPipeServerStream("MFHarnessPipe", PipeDirection.InOut, 1,
PipeTransmissionMode.Message, PipeOptions.WriteThrough);
}
// Set log file name.
SetLogFileName(test.Name);
test.LogFile = m_outputXmlFileName;
if (!(test is ProfilerTest))
{
m_log = log;
m_log.StartLog(m_outputXmlFileName, m_xslPath);
}
// Get build and exe path.
string buildPath = GetBuildPath(file);
test.ExeLocation = buildPath + m_assemblyName + ".exe";
// If this is a profiler test, run the test and return.
if (test is ProfilerTest)
{
return(RunProfilerTest(file, test.ExeLocation, buildPath, referenceList));
}
string testName = m_logFileName.Replace(".xml", string.Empty);
StartCodeCoverage(testName);
m_startTime = DateTime.Now;
switch (m_currentAppType)
{
case TestType.Device:
try
{
RunDeviceTest(buildPath, test.ExeLocation, referenceList);
}
catch (Exception ex)
{
Close();
return(HarnessResult(ex));
}
break;
case TestType.DeviceDesktop:
// Desktop apps under
// DevBox: %spoclient%\Test\Platform\Tests\Desktop\Applications\
// TestBox: %programfiles%\Microsoft .NET Micro Framework\<version>\Tests\Desktop\
desktopThread = StartDesktopApplication(pathForFile, ref csprojFilePath);
pipeStream.WaitForConnection();
// Get references and build the device test.
GetProjectReferences(file, ref csprojFilePath);
BuildTest(csprojFilePath, TestType.Device);
RunDeviceTest(buildPath, test.ExeLocation, referenceList);
break;
case TestType.Desktop:
RunDesktopTest(test.ExeLocation);
break;
}
m_endTime = DateTime.Now;
StopCodeCoverage(testName);
}
catch (Exception ex)
{
m_endTime = DateTime.Now;
if (null != m_log)
{
m_log.WriteElementString("Test_Exception", ex.Message + ex.StackTrace);
}
Utils.WriteToEventLog("Exception: " + ex.ToString());
Close();
if (ex is ApplicationException && ex.Message.ToLower().Contains("build failure"))
{
return(HarnessExecutionResult.Abort);
}
else
{
return(HarnessExecutionResult.Unavailable);
}
}
finally
{
if (m_currentAppType == TestType.DeviceDesktop)
{
StopDesktopApplication(pipeStream, desktopThread);
}
Close();
if (null != m_log)
{
switch (m_currentAppType)
{
case TestType.DeviceDesktop:
SaveLogFile(test.Location, TestType.DeviceDesktop);
break;
default:
SaveLogFile(test.Location);
break;
}
}
// Change test results back to unknown for the next test.
TestResults = Result.NotKnown;
}
if (!string.IsNullOrEmpty(file))
{
if (!string.Equals(m_transport.ToLower(), "emulator"))
{
m_log.SynchronizeLogTime(file, m_initialTime);
}
}
else
{
return(HarnessExecutionResult.Unavailable);
}
}
catch (Exception ex)
{
if (ex is FileNotFoundException)
{
throw ex;
}
Utils.WriteToEventLog(string.Format("Exception in Harness: {0}", ex.ToString()));
return(HarnessExecutionResult.Abort);
}
return(HarnessExecutionResult.Success);
}
/// <summary>
/// Overloaded Constructor
/// </summary>
/// <param name="test">The name of the sln file.</param>
/// <param name="harness">The Harness instance.</param>
/// <param name="log">Path to log file. </param>
internal TimedTest(BaseTest test, Harness harness, XmlLog log)
{
this.m_test = test;
this.m_harness = harness;
this.m_log = log;
}
