public void RunCompositeFollowedByRun() { TestPlan target = getTestTestPlan(); target.Open(); target.Execute(); target.Close(); Log.Flush(); TestTraceListener trace2 = new TestTraceListener(); Log.AddListener(trace2); target.Open(); target.Execute(); target.Close(); Log.RemoveListener(trace2); TestTraceListener trace1 = new TestTraceListener(); Log.AddListener(trace1); target.Execute(); Log.RemoveListener(trace1); string allLog1 = trace1.allLog.ToString(); string allLog2 = trace2.allLog.ToString(); string[] log1Lines = filterLog(allLog1); string[] log2Lines = filterLog(allLog2); Assert.AreEqual(log1Lines.Count() + 2, log2Lines.Count(), allLog2); for (int i = 0; i < log1Lines.Length; i++) { CollectionAssert.Contains(log2Lines, log1Lines[i]); } }
public void RunCompositeOpenClose() { TestPlan target = getTestTestPlan(); target.Open(); target.Close(); target.Open(); target.Close(); }
public void RunCompositeStartTime() { PlanRunCollectorListener listener = new PlanRunCollectorListener(); ResultSettings.Current.Add(listener); TestTraceListener trace = new TestTraceListener(); Log.AddListener(trace); TestPlan target = getTestTestPlan(); target.Open(); target.Execute(); listener.StepRuns.Clear(); target.Execute(); target.Close(); Log.RemoveListener(trace); ResultSettings.Current.Remove(listener); Assert.AreEqual(2, listener.PlanRuns.Select(run => run.StartTimeStamp).Distinct().Count()); Assert.AreEqual(1, listener.StepRuns.Count()); }
public void ResultListenerAbortPlan() { // since the resultListenerCrash[] crashers = new resultListenerCrash[] { new resultListenerCrash { CrashResultPhase = resultListenerCrash.ResultPhase.PlanRunStart, AbortPlan = true }, new resultListenerCrash { CrashResultPhase = resultListenerCrash.ResultPhase.StepRunStart, AbortPlan = true }, new resultListenerCrash { CrashResultPhase = resultListenerCrash.ResultPhase.Result, AbortPlan = true }, new resultListenerCrash { CrashResultPhase = resultListenerCrash.ResultPhase.StepRunCompleted, AbortPlan = true }, // The plan cannot be aborted on PlanRunCompleted. The at that point it will be ignored. new resultListenerCrash { CrashResultPhase = resultListenerCrash.ResultPhase.PlanRunCompleted, AbortPlan = true } }; TestPlan testplan = new TestPlan(); testplan.Steps.Add(new TestStepTest()); foreach (var c in crashers) { c.FinalVerdict = Verdict.NotSet; var expectedVerdict = Verdict.Aborted; if (c.CrashResultPhase != resultListenerCrash.ResultPhase.PlanRunStart) { expectedVerdict = Verdict.Pass; } // Simply running the plan. var planrun = testplan.Execute(new IResultListener[] { c }); Assert.AreEqual(expectedVerdict, planrun.Verdict); Assert.AreEqual(expectedVerdict, c.FinalVerdict); // Test that it works in composite runs. // Here it's important that the abort does not spill into the next run. // Which is why the plan is run twice. testplan.Open(new IResultListener[] { c }); c.FinalVerdict = Verdict.NotSet; Assert.AreEqual(expectedVerdict, testplan.Execute(new IResultListener[] { c }).Verdict); Assert.AreEqual(expectedVerdict, c.FinalVerdict); c.CrashResultPhase = resultListenerCrash.ResultPhase.None; c.FinalVerdict = Verdict.NotSet; Assert.AreEqual(Verdict.Pass, testplan.Execute(new IResultListener[] { c }).Verdict); Assert.AreEqual(Verdict.Pass, c.FinalVerdict); testplan.Close(); } }
public void RunCompositeRunTwice() { TestTraceListener trace = new TestTraceListener(); Log.AddListener(trace); TestPlan target = getTestTestPlan(); target.Open(); target.Execute(); target.Execute(); target.Close(); Log.RemoveListener(trace); trace.AssertErrors(); TestTestStep step = target.Steps[0] as TestTestStep; Assert.AreEqual(2, step.PrePlanRunCount, "PrePlanRun was not called the correct number of times."); Assert.AreEqual(2, step.PostPlanRunCount, "PostPlanRun was not called the correct number of times."); }
public void RunCompositeMetaData2() { PlanRunCollectorListener listener = new PlanRunCollectorListener(); ResultSettings.Current.Add(listener); TestTraceListener trace = new TestTraceListener(); Log.AddListener(trace); TestPlan target = getTestTestPlan(); target.Open(); target.Execute(); target.Execute(); target.Close(); Log.RemoveListener(trace); ResultSettings.Current.Remove(listener); Assert.IsTrue(listener.PlanRuns.First().Parameters.Any(par => par.Value.ToString() == "Test Instrument")); Assert.AreEqual(1, listener.PlanRuns.Last().Parameters.Count(par => par.Value.ToString() == "Test Instrument")); }
public void RunCompositeAddInstrumentAfterOpen() { PlanRunCollectorListener listener = new PlanRunCollectorListener(); ResultSettings.Current.Add(listener); TestTraceListener trace = new TestTraceListener(); Log.AddListener(trace); TestPlan target = getTestTestPlan(); target.Open(); TestInstrument instr = InstrumentSettings.Current.FirstOrDefault(i => i is TestInstrument && (i as TestInstrument).Name.EndsWith("2")) as TestInstrument; if (instr == null) { instr = new TestInstrument { Name = "Test Instrument 2" }; InstrumentSettings.Current.Add(instr); } (target.Steps[0] as TestTestStep).Instr = instr; target.Execute(); listener.StepRuns.Clear(); target.Execute(); target.Close(); Log.RemoveListener(trace); ResultSettings.Current.Remove(listener); Assert.AreEqual(2, listener.PlanRuns.Select(run => run.StartTimeStamp).Distinct().Count()); Assert.AreEqual(1, listener.StepRuns.Count()); }
public void RunCompositeLogComparison() { // ## Fragile Test // If the settings directory has been loaded in the meantime // Log messages that new files are being created will appear on first run // ComponentSettings are lazily loaded so that first happens when the test plan runs. // We do one run to clear the state of the engine. { TestPlan target2 = getTestTestPlan(); Log.Flush(); target2.PrintTestPlanRunSummary = true; target2.Execute(); Log.Flush(); } TestTraceListener trace1 = new TestTraceListener(); Log.AddListener(trace1); TestPlan target = getTestTestPlan(); target.PrintTestPlanRunSummary = true; target.Execute(); Log.RemoveListener(trace1); TestTraceListener trace2 = new TestTraceListener(); Log.AddListener(trace2); target = getTestTestPlan(); target.PrintTestPlanRunSummary = true; target.Open(); target.Execute(); target.Close(); Log.RemoveListener(trace2); string allLog1 = trace1.allLog.ToString(); string allLog2 = trace2.allLog.ToString(); string[] log1Lines = filterLog(allLog1); string[] log2Lines = filterLog(allLog2); string[] log2LinesNoSpaces = filterLog(allLog2, true); Assert.AreEqual(log1Lines.Count() + 2, log2Lines.Count(), allLog1 + Environment.NewLine + "##########" + Environment.NewLine + allLog2); for (int i = 0; i < log1Lines.Length; i++) { var line = log1Lines[i].Replace(" ", ""); if (!log2LinesNoSpaces.Contains(line)) // We compare lines with removed spaces to avoid flakyness in CI. { // Print actual comparison data Console.WriteLine($"Could not find '{line}' in following logs:"); foreach (var linez in log2LinesNoSpaces) { Console.WriteLine($"{linez}"); } Console.WriteLine($"--------------- Printing logs without spaces removed ---------------"); // Print log data without their spaces removed Console.WriteLine($"First run logs:"); foreach (var linez in log1Lines) { Console.WriteLine($"- {linez}"); } Console.WriteLine($"Second run logs:"); foreach (var linez in log2Lines) { Console.WriteLine($"- {linez}"); } Assert.Fail($"The logs from two testplan executions does not match..."); } } }