/// <summary>
/// used by the run fuction to run the tests
/// </summary>
/// <param name="runner"></param>
/// <param name="resultsFile"></param>
private void RunTests(IAssetRunner runner, string resultsFile)
{
try
{
if (_ciRun)
{
_xmlBuilder = new JunitXmlBuilder();
_xmlBuilder.XmlName = resultsFile;
}
TestSuiteRunResults results = runner.Run();
if (results == null)
{
Environment.Exit((int)Launcher.ExitCodeEnum.Failed);
}
_xmlBuilder.CreateXmlFromRunResults(results);
//if there is an error
if (results.TestRuns.Any(tr => tr.TestState == TestState.Failed || tr.TestState == TestState.Error))
{
Launcher.ExitCode = Launcher.ExitCodeEnum.Failed;
}
//this is the total run summary
ConsoleWriter.ActiveTestRun = null;
string runStatus = (Launcher.ExitCode == ExitCodeEnum.Passed || Launcher.ExitCode == ExitCodeEnum.Unstable) ? "Job succeeded" : "Job failed";
int numFailures = results.TestRuns.Count(t => t.TestState == TestState.Failed);
int numSuccess = results.TestRuns.Count(t => t.TestState == TestState.Passed);
int numErrors = results.TestRuns.Count(t => t.TestState == TestState.Error);
ConsoleWriter.WriteLine(Resources.LauncherDoubleSeperator);
ConsoleWriter.WriteLine(string.Format(Resources.LauncherDisplayStatistics, runStatus, results.TestRuns.Count, numSuccess, numFailures, numErrors));
if (!runner.RunWasCancelled)
{
results.TestRuns.ForEach(tr => ConsoleWriter.WriteLine(((tr.HasWarnings) ? "Warning".PadLeft(7) : tr.TestState.ToString().PadRight(7)) + ": " + tr.TestPath));
ConsoleWriter.WriteLine(Resources.LauncherDoubleSeperator);
if (ConsoleWriter.ErrorSummaryLines != null && ConsoleWriter.ErrorSummaryLines.Count > 0)
{
ConsoleWriter.WriteLine("Job Errors summary:");
ConsoleWriter.ErrorSummaryLines.ForEach(line => ConsoleWriter.WriteLine(line));
}
}
//ConsoleWriter.WriteLine("Returning " + runStatus + ".");
}
finally
{
try
{
runner.Dispose();
}
catch (Exception ex)
{
ConsoleWriter.WriteLine(string.Format(Resources.LauncherRunnerDisposeError, ex.Message));
};
}
}
/// <summary>
/// used by the run fuction to run the tests
/// </summary>
/// <param name="runner"></param>
/// <param name="resultsFile"></param>
private void RunTests(IAssetRunner runner, string resultsFile)
{
try
{
if (_ciRun)
{
_xmlBuilder = new JunitXmlBuilder();
_xmlBuilder.XmlName = resultsFile;
}
TestSuiteRunResults results = runner.Run();
if (results == null)
Environment.Exit((int)Launcher.ExitCodeEnum.Failed);
_xmlBuilder.CreateXmlFromRunResults(results);
//if there is an error
if (results.TestRuns.Any(tr => tr.TestState == TestState.Failed || tr.TestState == TestState.Error))
{
Launcher.ExitCode = Launcher.ExitCodeEnum.Failed;
}
//this is the total run summary
ConsoleWriter.ActiveTestRun = null;
string runStatus = (Launcher.ExitCode == ExitCodeEnum.Passed || Launcher.ExitCode == ExitCodeEnum.Unstable) ? "Job succeeded" : "Job failed";
int numFailures = results.TestRuns.Count(t => t.TestState == TestState.Failed);
int numSuccess = results.TestRuns.Count(t => t.TestState == TestState.Passed);
int numErrors = results.TestRuns.Count(t => t.TestState == TestState.Error);
ConsoleWriter.WriteLine(Resources.LauncherDoubleSeperator);
ConsoleWriter.WriteLine(string.Format(Resources.LauncherDisplayStatistics, runStatus, results.TestRuns.Count, numSuccess, numFailures, numErrors));
if (!runner.RunWasCancelled)
{
results.TestRuns.ForEach(tr => ConsoleWriter.WriteLine(((tr.HasWarnings) ? "Warning".PadLeft(7) : tr.TestState.ToString().PadRight(7)) + ": " + tr.TestPath));
ConsoleWriter.WriteLine(Resources.LauncherDoubleSeperator);
if (ConsoleWriter.ErrorSummaryLines != null && ConsoleWriter.ErrorSummaryLines.Count > 0)
{
ConsoleWriter.WriteLine("Job Errors summary:");
ConsoleWriter.ErrorSummaryLines.ForEach(line => ConsoleWriter.WriteLine(line));
}
}
//ConsoleWriter.WriteLine("Returning " + runStatus + ".");
}
finally
{
try
{
runner.Dispose();
}
catch (Exception ex)
{
ConsoleWriter.WriteLine(string.Format(Resources.LauncherRunnerDisposeError, ex.Message));
};
}
}
private void RunTests(IAssetRunner runner, string resultsFile, TestSuiteRunResults results)
{
try
{
if (_ciRun)
{
_xmlBuilder = new JunitXmlBuilder();
_xmlBuilder.XmlName = resultsFile;
}
if (results == null)
{
Environment.Exit((int)Launcher.ExitCodeEnum.Failed);
}
_xmlBuilder.CreateXmlFromRunResults(results);
if (results.TestRuns.Count == 0)
{
Console.WriteLine("No tests were run");
Launcher.ExitCode = Launcher.ExitCodeEnum.Failed;
Environment.Exit((int)Launcher.ExitCode);
}
//if there is an error
if (results.TestRuns.Any(tr => tr.TestState == TestState.Failed || tr.TestState == TestState.Error))
{
Launcher.ExitCode = Launcher.ExitCodeEnum.Failed;
}
int numFailures = results.TestRuns.Count(t => t.TestState == TestState.Failed);
int numSuccess = results.TestRuns.Count(t => t.TestState == TestState.Passed);
int numErrors = results.TestRuns.Count(t => t.TestState == TestState.Error);
int numWarnings = results.TestRuns.Count(t => t.TestState == TestState.Warning);
if ((numErrors <= 0) && (numFailures > 0))
{
Launcher.ExitCode = Launcher.ExitCodeEnum.Failed;
}
if ((numErrors <= 0) && (numFailures > 0) && (numSuccess > 0))
{
Launcher.ExitCode = Launcher.ExitCodeEnum.Unstable;
}
foreach (var testRun in results.TestRuns)
{
if (testRun.FatalErrors > 0 && !testRun.TestPath.Equals(""))
{
Launcher.ExitCode = Launcher.ExitCodeEnum.Failed;
break;
}
}
//this is the total run summary
ConsoleWriter.ActiveTestRun = null;
string runStatus = "";
switch (Launcher.ExitCode)
{
case ExitCodeEnum.Passed:
runStatus = "Job succeeded";
break;
case ExitCodeEnum.Unstable:
runStatus = "Job unstable (Passed with failed tests)";
break;
case ExitCodeEnum.Aborted:
runStatus = "Job failed due to being Aborted";
break;
case ExitCodeEnum.Failed:
runStatus = "Job failed";
break;
default:
runStatus = "Error: Job status is Undefined";
break;
}
ConsoleWriter.WriteLine(Resources.LauncherDoubleSeperator);
ConsoleWriter.WriteLine(string.Format(Resources.LauncherDisplayStatistics, runStatus, results.TestRuns.Count, numSuccess, numFailures, numErrors, numWarnings));
int testIndex = 1;
if (!runner.RunWasCancelled)
{
results.TestRuns.ForEach(tr => { ConsoleWriter.WriteLine(((tr.HasWarnings) ? "Warning".PadLeft(7) : tr.TestState.ToString().PadRight(7)) + ": " + tr.TestPath + "[" + testIndex + "]"); testIndex++; });
ConsoleWriter.WriteLine(Resources.LauncherDoubleSeperator);
if (ConsoleWriter.ErrorSummaryLines != null && ConsoleWriter.ErrorSummaryLines.Count > 0)
{
ConsoleWriter.WriteLine("Job Errors summary:");
ConsoleWriter.ErrorSummaryLines.ForEach(line => ConsoleWriter.WriteLine(line));
}
string onCheckFailedTests = (_ciParams.ContainsKey("onCheckFailedTest") ? _ciParams["onCheckFailedTest"] : "");
_rerunFailedTests = !string.IsNullOrEmpty(onCheckFailedTests) && Convert.ToBoolean(onCheckFailedTests.ToLower());
if (!_rerunFailedTests)
{
Environment.Exit((int)Launcher.ExitCode);
}
}
}
finally
{
try
{
runner.Dispose();
}
catch (Exception ex)
{
ConsoleWriter.WriteLine(string.Format(Resources.LauncherRunnerDisposeError, ex.Message));
};
}
}
/// <summary>
/// used by the run fuction to run the tests
/// </summary>
/// <param name="runner"></param>
/// <param name="resultsFile"></param>
private void RunTests(IAssetRunner runner, string resultsFile)
{
try
{
if (_ciRun)
{
_xmlBuilder = new JunitXmlBuilder();
_xmlBuilder.XmlName = resultsFile;
}
TestSuiteRunResults results = runner.Run();
if (results == null)
{
Environment.Exit((int)Launcher.ExitCodeEnum.Failed);
}
_xmlBuilder.CreateXmlFromRunResults(results);
//if there is an error
if (results.TestRuns.Any(tr => tr.TestState == TestState.Failed || tr.TestState == TestState.Error))
{
Launcher.ExitCode = Launcher.ExitCodeEnum.Failed;
}
int numFailures = results.TestRuns.Count(t => t.TestState == TestState.Failed);
int numSuccess = results.TestRuns.Count(t => t.TestState == TestState.Passed);
int numErrors = results.TestRuns.Count(t => t.TestState == TestState.Error);
//TODO: Temporery fix to remove since jenkins doesnt retrive resutls from jobs that marked as failed and unstable marks jobs with only failed tests
if ((numErrors <= 0) && (numFailures > 0))
{
Launcher.ExitCode = Launcher.ExitCodeEnum.Unstable;
}
foreach (var testRun in results.TestRuns)
{
if (testRun.FatalErrors > 0 && !testRun.TestPath.Equals(""))
{
Launcher.ExitCode = Launcher.ExitCodeEnum.Failed;
break;
}
}
//this is the total run summary
ConsoleWriter.ActiveTestRun = null;
string runStatus = "";
switch (Launcher.ExitCode)
{
case ExitCodeEnum.Passed:
runStatus = "Job succeeded";
break;
case ExitCodeEnum.Unstable:
runStatus = "Job unstable (Passed with failed tests)";
break;
case ExitCodeEnum.Aborted:
runStatus = "Job failed due to being Aborted";
break;
case ExitCodeEnum.Failed:
runStatus = "Job failed";
break;
default:
runStatus = "Error: Job status is Undefined";
break;
}
ConsoleWriter.WriteLine(Resources.LauncherDoubleSeperator);
ConsoleWriter.WriteLine(string.Format(Resources.LauncherDisplayStatistics, runStatus, results.TestRuns.Count, numSuccess, numFailures, numErrors));
if (!runner.RunWasCancelled)
{
results.TestRuns.ForEach(tr => ConsoleWriter.WriteLine(((tr.HasWarnings) ? "Warning".PadLeft(7) : tr.TestState.ToString().PadRight(7)) + ": " + tr.TestPath));
ConsoleWriter.WriteLine(Resources.LauncherDoubleSeperator);
if (ConsoleWriter.ErrorSummaryLines != null && ConsoleWriter.ErrorSummaryLines.Count > 0)
{
ConsoleWriter.WriteLine("Job Errors summary:");
ConsoleWriter.ErrorSummaryLines.ForEach(line => ConsoleWriter.WriteLine(line));
}
}
//ConsoleWriter.WriteLine("Returning " + runStatus + ".");
}
finally
{
try
{
runner.Dispose();
}
catch (Exception ex)
{
ConsoleWriter.WriteLine(string.Format(Resources.LauncherRunnerDisposeError, ex.Message));
};
}
}
/// <summary>
/// used by the run fuction to run the tests
/// </summary>
/// <param name="runner"></param>
/// <param name="resultsFile"></param>
private void RunTests(IAssetRunner runner, string resultsFile)
{
try
{
if (_ciRun)
{
_xmlBuilder = new JunitXmlBuilder();
_xmlBuilder.XmlName = resultsFile;
}
TestSuiteRunResults results = runner.Run();
if (results == null)
Environment.Exit((int)Launcher.ExitCodeEnum.Failed);
_xmlBuilder.CreateXmlFromRunResults(results);
//if there is an error
if (results.TestRuns.Any(tr => tr.TestState == TestState.Failed || tr.TestState == TestState.Error))
{
Launcher.ExitCode = Launcher.ExitCodeEnum.Failed;
}
int numFailures = results.TestRuns.Count(t => t.TestState == TestState.Failed);
int numSuccess = results.TestRuns.Count(t => t.TestState == TestState.Passed);
int numErrors = results.TestRuns.Count(t => t.TestState == TestState.Error);
//TODO: Temporery fix to remove since jenkins doesnt retrive resutls from jobs that marked as failed and unstable marks jobs with only failed tests
if ((numErrors <= 0) && (numFailures > 0))
{
Launcher.ExitCode = Launcher.ExitCodeEnum.Unstable;
}
foreach (var testRun in results.TestRuns)
{
if (testRun.FatalErrors > 0 && !testRun.TestPath.Equals(""))
{
Launcher.ExitCode = Launcher.ExitCodeEnum.Failed;
break;
}
}
//this is the total run summary
ConsoleWriter.ActiveTestRun = null;
string runStatus = "";
switch (Launcher.ExitCode)
{
case ExitCodeEnum.Passed:
runStatus = "Job succeeded";
break;
case ExitCodeEnum.Unstable:
runStatus = "Job unstable (Passed with failed tests)";
break;
case ExitCodeEnum.Aborted:
runStatus = "Job failed due to being Aborted";
break;
case ExitCodeEnum.Failed:
runStatus = "Job failed";
break;
default:
runStatus = "Error: Job status is Undefined";
break;
}
ConsoleWriter.WriteLine(Resources.LauncherDoubleSeperator);
ConsoleWriter.WriteLine(string.Format(Resources.LauncherDisplayStatistics, runStatus, results.TestRuns.Count, numSuccess, numFailures, numErrors));
if (!runner.RunWasCancelled)
{
results.TestRuns.ForEach(tr => ConsoleWriter.WriteLine(((tr.HasWarnings) ? "Warning".PadLeft(7) : tr.TestState.ToString().PadRight(7)) + ": " + tr.TestPath));
ConsoleWriter.WriteLine(Resources.LauncherDoubleSeperator);
if (ConsoleWriter.ErrorSummaryLines != null && ConsoleWriter.ErrorSummaryLines.Count > 0)
{
ConsoleWriter.WriteLine("Job Errors summary:");
ConsoleWriter.ErrorSummaryLines.ForEach(line => ConsoleWriter.WriteLine(line));
}
}
//ConsoleWriter.WriteLine("Returning " + runStatus + ".");
}
finally
{
try
{
runner.Dispose();
}
catch (Exception ex)
{
ConsoleWriter.WriteLine(string.Format(Resources.LauncherRunnerDisposeError, ex.Message));
};
}
}