private TestPlanRun DoExecute(IEnumerable <IResultListener> resultListeners, IEnumerable <ResultParameter> metaDataParameters, HashSet <ITestStep> stepsOverride)
{
if (resultListeners == null)
{
throw new ArgumentNullException("resultListeners");
}
if (PrintTestPlanRunSummary && !resultListeners.Contains(summaryListener))
{
resultListeners = resultListeners.Concat(new IResultListener[] { summaryListener });
}
resultListeners = resultListeners.Where(r => r is IEnabledResource ? ((IEnabledResource)r).IsEnabled : true);
IList <ITestStep> steps;
if (stepsOverride == null)
{
steps = Steps;
}
else
{
// Remove steps that are already included via their parent steps.
foreach (var step in stepsOverride)
{
if (step == null)
{
throw new ArgumentException("stepsOverride may not contain null", "stepsOverride");
}
var p = step.GetParent <ITestStep>();
while (p != null)
{
if (stepsOverride.Contains(p))
{
throw new ArgumentException("stepsOverride may not contain steps and their parents.", "stepsOverride");
}
p = p.GetParent <ITestStep>();
}
}
steps = Utils.FlattenHeirarchy(Steps, step => step.ChildTestSteps).Where(stepsOverride.Contains).ToList();
}
long initTimeStamp = Stopwatch.GetTimestamp();
var initTime = DateTime.Now;
Log.Info("-----------------------------------------------------------------");
var fileStreamFile = FileSystemHelper.CreateTempFile();
var logStream = new HybridStream(fileStreamFile, 1024 * 1024);
var planRunLog = new FileTraceListener(logStream)
{
IsRelative = true
};
OpenTap.Log.AddListener(planRunLog);
var allSteps = Utils.FlattenHeirarchy(steps, step => step.ChildTestSteps);
var allEnabledSteps = Utils.FlattenHeirarchy(steps.Where(x => x.Enabled), step => step.GetEnabledChildSteps());
var enabledSinks = new HashSet <IResultSink>();
TestStepExtensions.GetObjectSettings <IResultSink, ITestStep, IResultSink>(allEnabledSteps, true, null, enabledSinks);
if (enabledSinks.Count > 0)
{
var sinkListener = new ResultSinkListener(enabledSinks);
resultListeners = resultListeners.Append(sinkListener);
}
Log.Info("Starting TestPlan '{0}' on {1}, {2} of {3} TestSteps enabled.", Name, initTime, allEnabledSteps.Count, allSteps.Count);
// Reset step verdict.
foreach (var step in allSteps)
{
if (step.Verdict != Verdict.NotSet)
{
step.Verdict = Verdict.NotSet;
step.OnPropertyChanged("Verdict");
}
}
if (currentExecutionState != null)
{
// load result listeners that are _not_ used in the previous runs.
// otherwise they wont get opened later.
foreach (var rl in resultListeners)
{
if (!currentExecutionState.ResultListeners.Contains(rl))
{
currentExecutionState.ResultListeners.Add(rl);
}
}
}
var currentListeners = currentExecutionState != null ? currentExecutionState.ResultListeners : resultListeners;
TestPlanRun execStage;
bool continuedExecutionState = false;
if (currentExecutionState != null)
{
execStage = new TestPlanRun(currentExecutionState, initTime, initTimeStamp);
continuedExecutionState = true;
}
else
{
execStage = new TestPlanRun(this, resultListeners.ToList(), initTime, initTimeStamp);
execStage.Parameters.AddRange(PluginManager.GetPluginVersions(allEnabledSteps));
execStage.ResourceManager.ResourceOpened += r =>
{
execStage.Parameters.AddRange(PluginManager.GetPluginVersions(new List <object> {
r
}));
};
}
if (metaDataParameters != null)
{
execStage.Parameters.AddRange(metaDataParameters);
}
var prevExecutingPlanRun = executingPlanRun.LocalValue;
executingPlanRun.LocalValue = execStage;
CurrentRun = execStage;
failState runWentOk = failState.StartFail;
// ReSharper disable once InconsistentNaming
var preRun_Run_PostRunTimer = Stopwatch.StartNew();
try
{
execStage.FailedToStart = true; // Set it here in case OpenInternal throws an exception. Could happen if a step is missing an instrument
OpenInternal(execStage, continuedExecutionState, currentListeners.Cast <IResource>().ToList(), allEnabledSteps);
execStage.WaitForSerialization();
execStage.ResourceManager.BeginStep(execStage, this, TestPlanExecutionStage.Execute, TapThread.Current.AbortToken);
if (continuedExecutionState)
{ // Since resources are not opened, getting metadata cannot be done in the wait for resources continuation
// like shown in TestPlanRun. Instead we do it here.
foreach (var res in execStage.ResourceManager.Resources)
{
execStage.Parameters.AddRange(ResultParameters.GetMetadataFromObject(res));
}
}
runWentOk = failState.ExecFail; //important if test plan is aborted and runWentOk is never returned.
runWentOk = execTestPlan(execStage, steps);
}
catch (Exception e)
{
if (e is OperationCanceledException && execStage.MainThread.AbortToken.IsCancellationRequested)
{
Log.Warning(String.Format("TestPlan aborted. ({0})", e.Message));
execStage.UpgradeVerdict(Verdict.Aborted);
}
else if (e is ThreadAbortException)
{
// It seems this actually never happens.
Log.Warning("TestPlan aborted.");
execStage.UpgradeVerdict(Verdict.Aborted);
//Avoid entering the finally clause.
Thread.Sleep(500);
}
else if (e is System.ComponentModel.LicenseException)
{
Log.Error(e.Message);
execStage.UpgradeVerdict(Verdict.Error);
}
else
{
Log.Warning("TestPlan aborted.");
Log.Error(e.Message);
Log.Debug(e);
execStage.UpgradeVerdict(Verdict.Error);
}
}
finally
{
execStage.FailedToStart = (runWentOk == failState.StartFail);
try
{
finishTestPlanRun(execStage, preRun_Run_PostRunTimer, runWentOk, planRunLog, logStream);
}
catch (Exception ex)
{
Log.Error("Error while finishing TestPlan.");
Log.Debug(ex);
}
OpenTap.Log.RemoveListener(planRunLog);
planRunLog.Dispose();
logStream.Dispose();
File.Delete(fileStreamFile);
// Clean all test steps StepRun, otherwise the next test plan execution will be stuck at TestStep.DoRun at steps that does not have a cleared StepRun.
foreach (var step in allSteps)
{
step.StepRun = null;
}
executingPlanRun.LocalValue = prevExecutingPlanRun;
CurrentRun = prevExecutingPlanRun;
}
return(execStage);
}
void finishTestPlanRun(TestPlanRun run, Stopwatch testPlanTimer, failState runWentOk, TraceListener Logger, HybridStream logStream)
{
try
{
if (run != null)
{
if (runWentOk == failState.StartFail)
{
if (PrintTestPlanRunSummary)
{
summaryListener.OnTestPlanRunStart(run); // Call this to ensure that the correct planrun is being summarized
}
if (run.Verdict < Verdict.Aborted)
{
run.Verdict = Verdict.Error;
}
}
for (int i = run.StepsWithPrePlanRun.Count - 1; i >= 0; i--)
{
Stopwatch postTimer = Stopwatch.StartNew();
String stepPath = string.Empty;
try
{
ITestStep step = run.StepsWithPrePlanRun[i];
if ((step as TestStep)?.PrePostPlanRunUsed ?? true)
{
stepPath = step.GetStepPath();
run.AddTestStepStateUpdate(step.Id, null, StepState.PostPlanRun);
try
{
run.ResourceManager.BeginStep(run, step, TestPlanExecutionStage.PostPlanRun, TapThread.Current.AbortToken);
step.PlanRun = run;
try
{
step.PostPlanRun();
}
finally
{
run.ResourceManager.EndStep(step, TestPlanExecutionStage.PostPlanRun);
step.PlanRun = null;
}
}
finally
{
run.AddTestStepStateUpdate(step.Id, null, StepState.Idle);
}
Log.Debug(postTimer, "{0} PostPlanRun completed.", stepPath);
}
}
catch (Exception ex)
{
Log.Warning("Error during post plan run of {0}.", stepPath);
Log.Debug(ex);
}
}
run.Duration = testPlanTimer.Elapsed;
}
if (run != null)
{
try
{
// The open resource threads might throw exceptions. If they do we must
// Wait() for them to catch the exception.
// If the run was aborted after the open resource threads were started but
// before we wait for them (e.g. by an error in PrePlanRun), then we do it
// here.
run.ResourceManager.WaitUntilAllResourcesOpened(TapThread.Current.AbortToken);
}
catch (OperationCanceledException)
{
// Ignore this because this typically means that the wait was cancelled before.
// Just to be sure also upgrade verdict to aborted.
run.UpgradeVerdict(Verdict.Aborted);
}
catch (AggregateException e)
{
if (e.InnerExceptions.Count == 1)
{
Log.Error("Failed to open resource ({0})", e.GetInnerMostExceptionMessage());
Log.Debug(e);
}
else
{
Log.Error("Errors while opening resources:", e.GetInnerMostExceptionMessage());
foreach (Exception ie in e.InnerExceptions)
{
Log.Error(" {0}", ie.GetInnerMostExceptionMessage());
Log.Debug(ie);
}
}
}
if (PrintTestPlanRunSummary)
{
// wait for the summaryListener so the summary appears in the log file.
run.WaitForResultListener(summaryListener);
summaryListener.PrintSummary();
}
OpenTap.Log.Flush();
Logger.Flush();
logStream.Flush();
run.AddTestPlanCompleted(logStream, runWentOk != failState.StartFail);
run.ResourceManager.EndStep(this, TestPlanExecutionStage.Execute);
if (!run.IsCompositeRun)
{
run.ResourceManager.EndStep(this, TestPlanExecutionStage.Open);
}
}
}
finally
{
if (monitors != null)
{
foreach (var item in monitors)
{
item.ExitTestPlanRun(run);
}
}
}
}