コード例 #1
0
        public void RunCompositeFollowedByRun()
        {
            TestPlan target = getTestTestPlan();

            target.Open();
            target.Execute();
            target.Close();
            Log.Flush();
            TestTraceListener trace2 = new TestTraceListener();

            Log.AddListener(trace2);
            target.Open();
            target.Execute();
            target.Close();
            Log.RemoveListener(trace2);
            TestTraceListener trace1 = new TestTraceListener();

            Log.AddListener(trace1);
            target.Execute();
            Log.RemoveListener(trace1);

            string allLog1 = trace1.allLog.ToString();
            string allLog2 = trace2.allLog.ToString();

            string[] log1Lines = filterLog(allLog1);
            string[] log2Lines = filterLog(allLog2);


            Assert.AreEqual(log1Lines.Count() + 2, log2Lines.Count(), allLog2);
            for (int i = 0; i < log1Lines.Length; i++)
            {
                CollectionAssert.Contains(log2Lines, log1Lines[i]);
            }
        }
コード例 #2
0
        public void RunCompositeOpenClose()
        {
            TestPlan target = getTestTestPlan();

            target.Open();
            target.Close();
            target.Open();
            target.Close();
        }
コード例 #3
0
        public void RunCompositeStartTime()
        {
            PlanRunCollectorListener listener = new PlanRunCollectorListener();

            ResultSettings.Current.Add(listener);

            TestTraceListener trace = new TestTraceListener();

            Log.AddListener(trace);

            TestPlan target = getTestTestPlan();

            target.Open();
            target.Execute();
            listener.StepRuns.Clear();
            target.Execute();
            target.Close();

            Log.RemoveListener(trace);

            ResultSettings.Current.Remove(listener);

            Assert.AreEqual(2, listener.PlanRuns.Select(run => run.StartTimeStamp).Distinct().Count());
            Assert.AreEqual(1, listener.StepRuns.Count());
        }
コード例 #4
0
        public void ResultListenerAbortPlan()
        {
            // since the

            resultListenerCrash[] crashers = new resultListenerCrash[] {
                new resultListenerCrash {
                    CrashResultPhase = resultListenerCrash.ResultPhase.PlanRunStart, AbortPlan = true
                },
                new resultListenerCrash {
                    CrashResultPhase = resultListenerCrash.ResultPhase.StepRunStart, AbortPlan = true
                },
                new resultListenerCrash {
                    CrashResultPhase = resultListenerCrash.ResultPhase.Result, AbortPlan = true
                },
                new resultListenerCrash {
                    CrashResultPhase = resultListenerCrash.ResultPhase.StepRunCompleted, AbortPlan = true
                },
                // The plan cannot be aborted on PlanRunCompleted. The at that point it will be ignored.
                new resultListenerCrash {
                    CrashResultPhase = resultListenerCrash.ResultPhase.PlanRunCompleted, AbortPlan = true
                }
            };

            TestPlan testplan = new TestPlan();

            testplan.Steps.Add(new TestStepTest());
            foreach (var c in crashers)
            {
                c.FinalVerdict = Verdict.NotSet;
                var expectedVerdict = Verdict.Aborted;

                if (c.CrashResultPhase != resultListenerCrash.ResultPhase.PlanRunStart)
                {
                    expectedVerdict = Verdict.Pass;
                }

                // Simply running the plan.
                var planrun = testplan.Execute(new IResultListener[] { c });
                Assert.AreEqual(expectedVerdict, planrun.Verdict);
                Assert.AreEqual(expectedVerdict, c.FinalVerdict);

                // Test that it works in composite runs.
                // Here it's important that the abort does not spill into the next run.
                // Which is why the plan is run twice.
                testplan.Open(new IResultListener[] { c });
                c.FinalVerdict = Verdict.NotSet;
                Assert.AreEqual(expectedVerdict, testplan.Execute(new IResultListener[] { c }).Verdict);
                Assert.AreEqual(expectedVerdict, c.FinalVerdict);
                c.CrashResultPhase = resultListenerCrash.ResultPhase.None;
                c.FinalVerdict     = Verdict.NotSet;
                Assert.AreEqual(Verdict.Pass, testplan.Execute(new IResultListener[] { c }).Verdict);
                Assert.AreEqual(Verdict.Pass, c.FinalVerdict);
                testplan.Close();
            }
        }
コード例 #5
0
        public void RunCompositeRunTwice()
        {
            TestTraceListener trace = new TestTraceListener();

            Log.AddListener(trace);
            TestPlan target = getTestTestPlan();

            target.Open();
            target.Execute();
            target.Execute();
            target.Close();
            Log.RemoveListener(trace);

            trace.AssertErrors();

            TestTestStep step = target.Steps[0] as TestTestStep;

            Assert.AreEqual(2, step.PrePlanRunCount, "PrePlanRun was not called the correct number of times.");
            Assert.AreEqual(2, step.PostPlanRunCount, "PostPlanRun was not called the correct number of times.");
        }
コード例 #6
0
        public void RunCompositeMetaData2()
        {
            PlanRunCollectorListener listener = new PlanRunCollectorListener();

            ResultSettings.Current.Add(listener);

            TestTraceListener trace = new TestTraceListener();

            Log.AddListener(trace);

            TestPlan target = getTestTestPlan();

            target.Open();
            target.Execute();
            target.Execute();
            target.Close();
            Log.RemoveListener(trace);

            ResultSettings.Current.Remove(listener);

            Assert.IsTrue(listener.PlanRuns.First().Parameters.Any(par => par.Value.ToString() == "Test Instrument"));
            Assert.AreEqual(1, listener.PlanRuns.Last().Parameters.Count(par => par.Value.ToString() == "Test Instrument"));
        }
コード例 #7
0
        public void RunCompositeAddInstrumentAfterOpen()
        {
            PlanRunCollectorListener listener = new PlanRunCollectorListener();

            ResultSettings.Current.Add(listener);

            TestTraceListener trace = new TestTraceListener();

            Log.AddListener(trace);

            TestPlan target = getTestTestPlan();

            target.Open();

            TestInstrument instr = InstrumentSettings.Current.FirstOrDefault(i => i is TestInstrument && (i as TestInstrument).Name.EndsWith("2")) as TestInstrument;

            if (instr == null)
            {
                instr = new TestInstrument {
                    Name = "Test Instrument 2"
                };
                InstrumentSettings.Current.Add(instr);
            }
            (target.Steps[0] as TestTestStep).Instr = instr;

            target.Execute();
            listener.StepRuns.Clear();
            target.Execute();
            target.Close();

            Log.RemoveListener(trace);

            ResultSettings.Current.Remove(listener);

            Assert.AreEqual(2, listener.PlanRuns.Select(run => run.StartTimeStamp).Distinct().Count());
            Assert.AreEqual(1, listener.StepRuns.Count());
        }
コード例 #8
0
        public void RunCompositeLogComparison()
        {
            // ## Fragile Test
            // If the settings directory has been loaded in the meantime
            // Log messages that new files are being created will appear on first run
            // ComponentSettings are lazily loaded so that first happens when the test plan runs.
            // We do one run to clear the state of the engine.
            {
                TestPlan target2 = getTestTestPlan();
                Log.Flush();
                target2.PrintTestPlanRunSummary = true;
                target2.Execute();
                Log.Flush();
            }

            TestTraceListener trace1 = new TestTraceListener();

            Log.AddListener(trace1);
            TestPlan target = getTestTestPlan();

            target.PrintTestPlanRunSummary = true;
            target.Execute();
            Log.RemoveListener(trace1);

            TestTraceListener trace2 = new TestTraceListener();

            Log.AddListener(trace2);
            target = getTestTestPlan();
            target.PrintTestPlanRunSummary = true;
            target.Open();
            target.Execute();
            target.Close();
            Log.RemoveListener(trace2);

            string allLog1 = trace1.allLog.ToString();
            string allLog2 = trace2.allLog.ToString();

            string[] log1Lines = filterLog(allLog1);
            string[] log2Lines = filterLog(allLog2);

            string[] log2LinesNoSpaces = filterLog(allLog2, true);

            Assert.AreEqual(log1Lines.Count() + 2, log2Lines.Count(), allLog1 + Environment.NewLine + "##########" + Environment.NewLine + allLog2);
            for (int i = 0; i < log1Lines.Length; i++)
            {
                var line = log1Lines[i].Replace(" ", "");
                if (!log2LinesNoSpaces.Contains(line)) // We compare lines with removed spaces to avoid flakyness in CI.
                {
                    // Print actual comparison data
                    Console.WriteLine($"Could not find '{line}' in following logs:");
                    foreach (var linez in log2LinesNoSpaces)
                    {
                        Console.WriteLine($"{linez}");
                    }

                    Console.WriteLine($"--------------- Printing logs without spaces removed ---------------");

                    // Print log data without their spaces removed
                    Console.WriteLine($"First run logs:");
                    foreach (var linez in log1Lines)
                    {
                        Console.WriteLine($"- {linez}");
                    }

                    Console.WriteLine($"Second run logs:");
                    foreach (var linez in log2Lines)
                    {
                        Console.WriteLine($"- {linez}");
                    }

                    Assert.Fail($"The logs from two testplan executions does not match...");
                }
            }
        }