/// <summary>
/// Gets the name and value for each metadata property in this object that is not null.
/// </summary>
/// <returns>Name and value for each metadata property in this
/// object that is not null as a MetaDataParameter object.</returns>
public static List <MetaDataParameter> GetMetaDataParameters(object obj)
{
if (obj == null)
{
throw new ArgumentNullException("obj");
}
var metadata = ResultParameters.GetMetadataFromObject(obj);
return(metadata.Select(meta => new MetaDataParameter(meta.Name, meta.Value)).ToList());
}
failState execTestPlan(TestPlanRun execStage, IList <ITestStep> steps)
{
WaitHandle.WaitAny(new[] { execStage.PromptWaitHandle, TapThread.Current.AbortToken.WaitHandle });
bool resultListenerError = false;
execStage.ScheduleInResultProcessingThread <IResultListener>(resultListener =>
{
try
{
using (TimeoutOperation.Create(() => PrintWaitingMessage(new List <IResource>()
{
resultListener
})))
execStage.ResourceManager.WaitUntilResourcesOpened(TapThread.Current.AbortToken, resultListener);
try
{
// some resources might set metadata in the Open methods.
// this information needs to be propagated to result listeners as well.
// this returns quickly if its a lazy resource manager.
using (TimeoutOperation.Create(
() => PrintWaitingMessage(new List <IResource>()
{
resultListener
})))
execStage.ResourceManager.WaitUntilAllResourcesOpened(TapThread.Current.AbortToken);
}
catch // this error will also be handled somewhere else.
{
}
execStage.WaitForSerialization();
foreach (var res in execStage.PromptedResources)
{
execStage.Parameters.AddRange(ResultParameters.GetMetadataFromObject(res));
}
resultListener.OnTestPlanRunStart(execStage);
}
catch (OperationCanceledException) when(execStage.MainThread.AbortToken.IsCancellationRequested)
{
// test plan thread was aborted, this is OK.
}
catch (Exception ex)
{
Log.Error("Error in OnTestPlanRunStart for '{0}': '{1}'", resultListener, ex.Message);
Log.Debug(ex);
resultListenerError = true;
}
}, true);
if (resultListenerError)
{
return(failState.StartFail);
}
var sw = Stopwatch.StartNew();
try
{
execStage.StepsWithPrePlanRun.Clear();
if (!runPrePlanRunMethods(steps, execStage))
{
return(failState.StartFail);
}
}
catch (Exception e)
{
Log.Error(e.GetInnerMostExceptionMessage());
Log.Debug(e);
return(failState.StartFail);
}
finally{
{
Log.Debug(sw, "PrePlanRun Methods completed");
}
}
Stopwatch planRunOnlyTimer = Stopwatch.StartNew();
var runs = new List <TestStepRun>();
try
{
for (int i = 0; i < steps.Count; i++)
{
var step = steps[i];
if (step.Enabled == false)
{
continue;
}
var run = step.DoRun(execStage, execStage);
if (!run.Skipped)
{
runs.Add(run);
}
run.CheckBreakCondition();
// note: The following is copied inside TestStep.cs
if (run.SuggestedNextStep is Guid id)
{
int nextindex = steps.IndexWhen(x => x.Id == id);
if (nextindex >= 0)
{
i = nextindex - 1;
}
// if skip to next step, dont add it to the wait queue.
}
}
}
catch (TestStepBreakException breakEx)
{
Log.Info("{0}", breakEx.Message);
}
finally
{
// Now wait for them to actually complete. They might defer internally.
foreach (var run in runs)
{
run.WaitForCompletion();
execStage.UpgradeVerdict(run.Verdict);
}
}
Log.Debug(planRunOnlyTimer, "Test step runs finished.");
return(failState.Ok);
}
private TestPlanRun DoExecute(IEnumerable <IResultListener> resultListeners, IEnumerable <ResultParameter> metaDataParameters, HashSet <ITestStep> stepsOverride)
{
if (resultListeners == null)
{
throw new ArgumentNullException("resultListeners");
}
if (PrintTestPlanRunSummary && !resultListeners.Contains(summaryListener))
{
resultListeners = resultListeners.Concat(new IResultListener[] { summaryListener });
}
resultListeners = resultListeners.Where(r => r is IEnabledResource ? ((IEnabledResource)r).IsEnabled : true);
IList <ITestStep> steps;
if (stepsOverride == null)
{
steps = Steps;
}
else
{
// Remove steps that are already included via their parent steps.
foreach (var step in stepsOverride)
{
if (step == null)
{
throw new ArgumentException("stepsOverride may not contain null", "stepsOverride");
}
var p = step.GetParent <ITestStep>();
while (p != null)
{
if (stepsOverride.Contains(p))
{
throw new ArgumentException("stepsOverride may not contain steps and their parents.", "stepsOverride");
}
p = p.GetParent <ITestStep>();
}
}
steps = Utils.FlattenHeirarchy(Steps, step => step.ChildTestSteps).Where(stepsOverride.Contains).ToList();
}
long initTimeStamp = Stopwatch.GetTimestamp();
var initTime = DateTime.Now;
Log.Info("-----------------------------------------------------------------");
var fileStreamFile = FileSystemHelper.CreateTempFile();
var logStream = new HybridStream(fileStreamFile, 1024 * 1024);
var planRunLog = new FileTraceListener(logStream)
{
IsRelative = true
};
OpenTap.Log.AddListener(planRunLog);
var allSteps = Utils.FlattenHeirarchy(steps, step => step.ChildTestSteps);
var allEnabledSteps = Utils.FlattenHeirarchy(steps.Where(x => x.Enabled), step => step.GetEnabledChildSteps());
var enabledSinks = new HashSet <IResultSink>();
TestStepExtensions.GetObjectSettings <IResultSink, ITestStep, IResultSink>(allEnabledSteps, true, null, enabledSinks);
if (enabledSinks.Count > 0)
{
var sinkListener = new ResultSinkListener(enabledSinks);
resultListeners = resultListeners.Append(sinkListener);
}
Log.Info("Starting TestPlan '{0}' on {1}, {2} of {3} TestSteps enabled.", Name, initTime, allEnabledSteps.Count, allSteps.Count);
// Reset step verdict.
foreach (var step in allSteps)
{
if (step.Verdict != Verdict.NotSet)
{
step.Verdict = Verdict.NotSet;
step.OnPropertyChanged("Verdict");
}
}
if (currentExecutionState != null)
{
// load result listeners that are _not_ used in the previous runs.
// otherwise they wont get opened later.
foreach (var rl in resultListeners)
{
if (!currentExecutionState.ResultListeners.Contains(rl))
{
currentExecutionState.ResultListeners.Add(rl);
}
}
}
var currentListeners = currentExecutionState != null ? currentExecutionState.ResultListeners : resultListeners;
TestPlanRun execStage;
bool continuedExecutionState = false;
if (currentExecutionState != null)
{
execStage = new TestPlanRun(currentExecutionState, initTime, initTimeStamp);
continuedExecutionState = true;
}
else
{
execStage = new TestPlanRun(this, resultListeners.ToList(), initTime, initTimeStamp);
execStage.Parameters.AddRange(PluginManager.GetPluginVersions(allEnabledSteps));
execStage.ResourceManager.ResourceOpened += r =>
{
execStage.Parameters.AddRange(PluginManager.GetPluginVersions(new List <object> {
r
}));
};
}
if (metaDataParameters != null)
{
execStage.Parameters.AddRange(metaDataParameters);
}
var prevExecutingPlanRun = executingPlanRun.LocalValue;
executingPlanRun.LocalValue = execStage;
CurrentRun = execStage;
failState runWentOk = failState.StartFail;
// ReSharper disable once InconsistentNaming
var preRun_Run_PostRunTimer = Stopwatch.StartNew();
try
{
execStage.FailedToStart = true; // Set it here in case OpenInternal throws an exception. Could happen if a step is missing an instrument
OpenInternal(execStage, continuedExecutionState, currentListeners.Cast <IResource>().ToList(), allEnabledSteps);
execStage.WaitForSerialization();
execStage.ResourceManager.BeginStep(execStage, this, TestPlanExecutionStage.Execute, TapThread.Current.AbortToken);
if (continuedExecutionState)
{ // Since resources are not opened, getting metadata cannot be done in the wait for resources continuation
// like shown in TestPlanRun. Instead we do it here.
foreach (var res in execStage.ResourceManager.Resources)
{
execStage.Parameters.AddRange(ResultParameters.GetMetadataFromObject(res));
}
}
runWentOk = failState.ExecFail; //important if test plan is aborted and runWentOk is never returned.
runWentOk = execTestPlan(execStage, steps);
}
catch (Exception e)
{
if (e is OperationCanceledException && execStage.MainThread.AbortToken.IsCancellationRequested)
{
Log.Warning(String.Format("TestPlan aborted. ({0})", e.Message));
execStage.UpgradeVerdict(Verdict.Aborted);
}
else if (e is ThreadAbortException)
{
// It seems this actually never happens.
Log.Warning("TestPlan aborted.");
execStage.UpgradeVerdict(Verdict.Aborted);
//Avoid entering the finally clause.
Thread.Sleep(500);
}
else if (e is System.ComponentModel.LicenseException)
{
Log.Error(e.Message);
execStage.UpgradeVerdict(Verdict.Error);
}
else
{
Log.Warning("TestPlan aborted.");
Log.Error(e.Message);
Log.Debug(e);
execStage.UpgradeVerdict(Verdict.Error);
}
}
finally
{
execStage.FailedToStart = (runWentOk == failState.StartFail);
try
{
finishTestPlanRun(execStage, preRun_Run_PostRunTimer, runWentOk, planRunLog, logStream);
}
catch (Exception ex)
{
Log.Error("Error while finishing TestPlan.");
Log.Debug(ex);
}
OpenTap.Log.RemoveListener(planRunLog);
planRunLog.Dispose();
logStream.Dispose();
File.Delete(fileStreamFile);
// Clean all test steps StepRun, otherwise the next test plan execution will be stuck at TestStep.DoRun at steps that does not have a cleared StepRun.
foreach (var step in allSteps)
{
step.StepRun = null;
}
executingPlanRun.LocalValue = prevExecutingPlanRun;
CurrentRun = prevExecutingPlanRun;
}
return(execStage);
}
/// <summary> Expands the text. Macros are harvested from the optional TestPlanRun or the test step.</summary>
/// <param name="run">A place to find additional metadata for macro expansion.</param>
/// <param name="date">If no date was found in the metadata, this date will be used. If date is not supplied, DateTime.Now will be used.</param>
/// <param name="testPlanDir">If no TestPlanDir was found in the metata, this TestPlanDir will be used.</param>
/// <param name="replacements">Overrides other macro parameters.</param>
/// <returns>The expanded string.</returns>
public string Expand(TestPlanRun run, DateTime?date, string testPlanDir, Dictionary <string, object> replacements)
{
ITestStepParent context = Context;
IEnumerable <(string, object)> getMacro()
{
// note: macros are case-insensitive.
if (testPlanDir != null)
{
yield return("TestPlanDir", testPlanDir);
}
if (date != null)
{
yield return("date", date);
}
if (replacements != null)
{
foreach (var elem in replacements)
{
yield return(elem.Key, elem.Value);
}
}
if (run != null)
{
var runparams = run.Parameters.Concat(ResultParameters.GetMetadataFromObject(run)).Where(y => y.IsMetaData);
foreach (var v in runparams)
{
var path = v.Value;
yield return(v.Name, path);
yield return(v.MacroName, path);
}
}
ITestStepParent ctx = context;
while (ctx != null)
{
var p = ResultParameters.GetMetadataFromObject(ctx);
foreach (var v in p)
{
if (v.IsMetaData == false)
{
continue;
}
var path = v.Value;
yield return(v.Name, path);
yield return(v.MacroName, path);
}
ctx = ctx.Parent;
}
yield return("date", DateTime.Now);
yield return("Verdict", Verdict.NotSet);
var met = ResultParameters.GetComponentSettingsMetadataLazy(false);
foreach (var ps in met)
{
foreach (var v in ps)
{
if (v.IsMetaData == false)
{
continue;
}
var path = v.Value;
yield return(v.Name, path);
yield return(v.MacroName, path);
}
}
}
return(ReplaceMacros(Text, getMacro().Select(x => (x.Item1, StringConvertProvider.GetString(x.Item2)))));
}