Ejemplo n.º 1
0
        private TestPlanRun DoExecute(IEnumerable <IResultListener> resultListeners, IEnumerable <ResultParameter> metaDataParameters, HashSet <ITestStep> stepsOverride)
        {
            if (resultListeners == null)
            {
                throw new ArgumentNullException("resultListeners");
            }

            if (PrintTestPlanRunSummary && !resultListeners.Contains(summaryListener))
            {
                resultListeners = resultListeners.Concat(new IResultListener[] { summaryListener });
            }
            resultListeners = resultListeners.Where(r => r is IEnabledResource ? ((IEnabledResource)r).IsEnabled : true);
            IList <ITestStep> steps;

            if (stepsOverride == null)
            {
                steps = Steps;
            }
            else
            {
                // Remove steps that are already included via their parent steps.
                foreach (var step in stepsOverride)
                {
                    if (step == null)
                    {
                        throw new ArgumentException("stepsOverride may not contain null", "stepsOverride");
                    }

                    var p = step.GetParent <ITestStep>();
                    while (p != null)
                    {
                        if (stepsOverride.Contains(p))
                        {
                            throw new ArgumentException("stepsOverride may not contain steps and their parents.", "stepsOverride");
                        }
                        p = p.GetParent <ITestStep>();
                    }
                }
                steps = Utils.FlattenHeirarchy(Steps, step => step.ChildTestSteps).Where(stepsOverride.Contains).ToList();
            }

            long initTimeStamp = Stopwatch.GetTimestamp();
            var  initTime      = DateTime.Now;

            Log.Info("-----------------------------------------------------------------");

            var fileStreamFile = FileSystemHelper.CreateTempFile();
            var logStream      = new HybridStream(fileStreamFile, 1024 * 1024);

            var planRunLog = new FileTraceListener(logStream)
            {
                IsRelative = true
            };

            OpenTap.Log.AddListener(planRunLog);

            var allSteps        = Utils.FlattenHeirarchy(steps, step => step.ChildTestSteps);
            var allEnabledSteps = Utils.FlattenHeirarchy(steps.Where(x => x.Enabled), step => step.GetEnabledChildSteps());

            var enabledSinks = new HashSet <IResultSink>();

            TestStepExtensions.GetObjectSettings <IResultSink, ITestStep, IResultSink>(allEnabledSteps, true, null, enabledSinks);
            if (enabledSinks.Count > 0)
            {
                var sinkListener = new ResultSinkListener(enabledSinks);
                resultListeners = resultListeners.Append(sinkListener);
            }

            Log.Info("Starting TestPlan '{0}' on {1}, {2} of {3} TestSteps enabled.", Name, initTime, allEnabledSteps.Count, allSteps.Count);

            // Reset step verdict.
            foreach (var step in allSteps)
            {
                if (step.Verdict != Verdict.NotSet)
                {
                    step.Verdict = Verdict.NotSet;
                    step.OnPropertyChanged("Verdict");
                }
            }

            if (currentExecutionState != null)
            {
                // load result listeners that are _not_ used in the previous runs.
                // otherwise they wont get opened later.
                foreach (var rl in resultListeners)
                {
                    if (!currentExecutionState.ResultListeners.Contains(rl))
                    {
                        currentExecutionState.ResultListeners.Add(rl);
                    }
                }
            }

            var currentListeners = currentExecutionState != null ? currentExecutionState.ResultListeners : resultListeners;

            TestPlanRun execStage;
            bool        continuedExecutionState = false;

            if (currentExecutionState != null)
            {
                execStage = new TestPlanRun(currentExecutionState, initTime, initTimeStamp);

                continuedExecutionState = true;
            }
            else
            {
                execStage = new TestPlanRun(this, resultListeners.ToList(), initTime, initTimeStamp);

                execStage.Parameters.AddRange(PluginManager.GetPluginVersions(allEnabledSteps));
                execStage.ResourceManager.ResourceOpened += r =>
                {
                    execStage.Parameters.AddRange(PluginManager.GetPluginVersions(new List <object> {
                        r
                    }));
                };
            }


            if (metaDataParameters != null)
            {
                execStage.Parameters.AddRange(metaDataParameters);
            }

            var prevExecutingPlanRun = executingPlanRun.LocalValue;

            executingPlanRun.LocalValue = execStage;
            CurrentRun = execStage;

            failState runWentOk = failState.StartFail;

            // ReSharper disable once InconsistentNaming
            var preRun_Run_PostRunTimer = Stopwatch.StartNew();

            try
            {
                execStage.FailedToStart = true; // Set it here in case OpenInternal throws an exception. Could happen if a step is missing an instrument

                OpenInternal(execStage, continuedExecutionState, currentListeners.Cast <IResource>().ToList(), allEnabledSteps);

                execStage.WaitForSerialization();
                execStage.ResourceManager.BeginStep(execStage, this, TestPlanExecutionStage.Execute, TapThread.Current.AbortToken);

                if (continuedExecutionState)
                {  // Since resources are not opened, getting metadata cannot be done in the wait for resources continuation
                   // like shown in TestPlanRun. Instead we do it here.
                    foreach (var res in execStage.ResourceManager.Resources)
                    {
                        execStage.Parameters.AddRange(ResultParameters.GetMetadataFromObject(res));
                    }
                }

                runWentOk = failState.ExecFail; //important if test plan is aborted and runWentOk is never returned.
                runWentOk = execTestPlan(execStage, steps);
            }
            catch (Exception e)
            {
                if (e is OperationCanceledException && execStage.MainThread.AbortToken.IsCancellationRequested)
                {
                    Log.Warning(String.Format("TestPlan aborted. ({0})", e.Message));
                    execStage.UpgradeVerdict(Verdict.Aborted);
                }
                else if (e is ThreadAbortException)
                {
                    // It seems this actually never happens.
                    Log.Warning("TestPlan aborted.");
                    execStage.UpgradeVerdict(Verdict.Aborted);
                    //Avoid entering the finally clause.
                    Thread.Sleep(500);
                }
                else if (e is System.ComponentModel.LicenseException)
                {
                    Log.Error(e.Message);
                    execStage.UpgradeVerdict(Verdict.Error);
                }
                else
                {
                    Log.Warning("TestPlan aborted.");
                    Log.Error(e.Message);
                    Log.Debug(e);
                    execStage.UpgradeVerdict(Verdict.Error);
                }
            }
            finally
            {
                execStage.FailedToStart = (runWentOk == failState.StartFail);

                try
                {
                    finishTestPlanRun(execStage, preRun_Run_PostRunTimer, runWentOk, planRunLog, logStream);
                }
                catch (Exception ex)
                {
                    Log.Error("Error while finishing TestPlan.");
                    Log.Debug(ex);
                }

                OpenTap.Log.RemoveListener(planRunLog);
                planRunLog.Dispose();

                logStream.Dispose();
                File.Delete(fileStreamFile);

                // Clean all test steps StepRun, otherwise the next test plan execution will be stuck at TestStep.DoRun at steps that does not have a cleared StepRun.
                foreach (var step in allSteps)
                {
                    step.StepRun = null;
                }

                executingPlanRun.LocalValue = prevExecutingPlanRun;
                CurrentRun = prevExecutingPlanRun;
            }
            return(execStage);
        }
Ejemplo n.º 2
0
        void finishTestPlanRun(TestPlanRun run, Stopwatch testPlanTimer, failState runWentOk, TraceListener Logger, HybridStream logStream)
        {
            try
            {
                if (run != null)
                {
                    if (runWentOk == failState.StartFail)
                    {
                        if (PrintTestPlanRunSummary)
                        {
                            summaryListener.OnTestPlanRunStart(run); // Call this to ensure that the correct planrun is being summarized
                        }
                        if (run.Verdict < Verdict.Aborted)
                        {
                            run.Verdict = Verdict.Error;
                        }
                    }

                    for (int i = run.StepsWithPrePlanRun.Count - 1; i >= 0; i--)
                    {
                        Stopwatch postTimer = Stopwatch.StartNew();
                        String    stepPath  = string.Empty;
                        try
                        {
                            ITestStep step = run.StepsWithPrePlanRun[i];

                            if ((step as TestStep)?.PrePostPlanRunUsed ?? true)
                            {
                                stepPath = step.GetStepPath();
                                run.AddTestStepStateUpdate(step.Id, null, StepState.PostPlanRun);
                                try
                                {
                                    run.ResourceManager.BeginStep(run, step, TestPlanExecutionStage.PostPlanRun, TapThread.Current.AbortToken);
                                    step.PlanRun = run;
                                    try
                                    {
                                        step.PostPlanRun();
                                    }
                                    finally
                                    {
                                        run.ResourceManager.EndStep(step, TestPlanExecutionStage.PostPlanRun);
                                        step.PlanRun = null;
                                    }
                                }
                                finally
                                {
                                    run.AddTestStepStateUpdate(step.Id, null, StepState.Idle);
                                }
                                Log.Debug(postTimer, "{0} PostPlanRun completed.", stepPath);
                            }
                        }
                        catch (Exception ex)
                        {
                            Log.Warning("Error during post plan run of {0}.", stepPath);
                            Log.Debug(ex);
                        }
                    }
                    run.Duration = testPlanTimer.Elapsed;
                }

                if (run != null)
                {
                    try
                    {
                        // The open resource threads might throw exceptions. If they do we must
                        // Wait() for them to catch the exception.
                        // If the run was aborted after the open resource threads were started but
                        // before we wait for them (e.g. by an error in PrePlanRun), then we do it
                        // here.
                        run.ResourceManager.WaitUntilAllResourcesOpened(TapThread.Current.AbortToken);
                    }
                    catch (OperationCanceledException)
                    {
                        // Ignore this because this typically means that the wait was cancelled before.
                        // Just to be sure also upgrade verdict to aborted.
                        run.UpgradeVerdict(Verdict.Aborted);
                    }
                    catch (AggregateException e)
                    {
                        if (e.InnerExceptions.Count == 1)
                        {
                            Log.Error("Failed to open resource ({0})", e.GetInnerMostExceptionMessage());
                            Log.Debug(e);
                        }
                        else
                        {
                            Log.Error("Errors while opening resources:", e.GetInnerMostExceptionMessage());
                            foreach (Exception ie in e.InnerExceptions)
                            {
                                Log.Error("    {0}", ie.GetInnerMostExceptionMessage());
                                Log.Debug(ie);
                            }
                        }
                    }

                    if (PrintTestPlanRunSummary)
                    {
                        // wait for the summaryListener so the summary appears in the log file.
                        run.WaitForResultListener(summaryListener);
                        summaryListener.PrintSummary();
                    }

                    OpenTap.Log.Flush();
                    Logger.Flush();
                    logStream.Flush();

                    run.AddTestPlanCompleted(logStream, runWentOk != failState.StartFail);

                    run.ResourceManager.EndStep(this, TestPlanExecutionStage.Execute);

                    if (!run.IsCompositeRun)
                    {
                        run.ResourceManager.EndStep(this, TestPlanExecutionStage.Open);
                    }
                }
            }
            finally
            {
                if (monitors != null)
                {
                    foreach (var item in monitors)
                    {
                        item.ExitTestPlanRun(run);
                    }
                }
            }
        }