public void RunCompositeFollowedByRun()
{
TestPlan target = getTestTestPlan();
target.Open();
target.Execute();
target.Close();
Log.Flush();
TestTraceListener trace2 = new TestTraceListener();
Log.AddListener(trace2);
target.Open();
target.Execute();
target.Close();
Log.RemoveListener(trace2);
TestTraceListener trace1 = new TestTraceListener();
Log.AddListener(trace1);
target.Execute();
Log.RemoveListener(trace1);
string allLog1 = trace1.allLog.ToString();
string allLog2 = trace2.allLog.ToString();
string[] log1Lines = filterLog(allLog1);
string[] log2Lines = filterLog(allLog2);
Assert.AreEqual(log1Lines.Count() + 2, log2Lines.Count(), allLog2);
for (int i = 0; i < log1Lines.Length; i++)
{
CollectionAssert.Contains(log2Lines, log1Lines[i]);
}
}
public void RunCompositeOpenClose()
{
TestPlan target = getTestTestPlan();
target.Open();
target.Close();
target.Open();
target.Close();
}
public void RunCompositeStartTime()
{
PlanRunCollectorListener listener = new PlanRunCollectorListener();
ResultSettings.Current.Add(listener);
TestTraceListener trace = new TestTraceListener();
Log.AddListener(trace);
TestPlan target = getTestTestPlan();
target.Open();
target.Execute();
listener.StepRuns.Clear();
target.Execute();
target.Close();
Log.RemoveListener(trace);
ResultSettings.Current.Remove(listener);
Assert.AreEqual(2, listener.PlanRuns.Select(run => run.StartTimeStamp).Distinct().Count());
Assert.AreEqual(1, listener.StepRuns.Count());
}
public void ResultListenerAbortPlan()
{
// since the
resultListenerCrash[] crashers = new resultListenerCrash[] {
new resultListenerCrash {
CrashResultPhase = resultListenerCrash.ResultPhase.PlanRunStart, AbortPlan = true
},
new resultListenerCrash {
CrashResultPhase = resultListenerCrash.ResultPhase.StepRunStart, AbortPlan = true
},
new resultListenerCrash {
CrashResultPhase = resultListenerCrash.ResultPhase.Result, AbortPlan = true
},
new resultListenerCrash {
CrashResultPhase = resultListenerCrash.ResultPhase.StepRunCompleted, AbortPlan = true
},
// The plan cannot be aborted on PlanRunCompleted. The at that point it will be ignored.
new resultListenerCrash {
CrashResultPhase = resultListenerCrash.ResultPhase.PlanRunCompleted, AbortPlan = true
}
};
TestPlan testplan = new TestPlan();
testplan.Steps.Add(new TestStepTest());
foreach (var c in crashers)
{
c.FinalVerdict = Verdict.NotSet;
var expectedVerdict = Verdict.Aborted;
if (c.CrashResultPhase != resultListenerCrash.ResultPhase.PlanRunStart)
{
expectedVerdict = Verdict.Pass;
}
// Simply running the plan.
var planrun = testplan.Execute(new IResultListener[] { c });
Assert.AreEqual(expectedVerdict, planrun.Verdict);
Assert.AreEqual(expectedVerdict, c.FinalVerdict);
// Test that it works in composite runs.
// Here it's important that the abort does not spill into the next run.
// Which is why the plan is run twice.
testplan.Open(new IResultListener[] { c });
c.FinalVerdict = Verdict.NotSet;
Assert.AreEqual(expectedVerdict, testplan.Execute(new IResultListener[] { c }).Verdict);
Assert.AreEqual(expectedVerdict, c.FinalVerdict);
c.CrashResultPhase = resultListenerCrash.ResultPhase.None;
c.FinalVerdict = Verdict.NotSet;
Assert.AreEqual(Verdict.Pass, testplan.Execute(new IResultListener[] { c }).Verdict);
Assert.AreEqual(Verdict.Pass, c.FinalVerdict);
testplan.Close();
}
}
public void RunCompositeRunTwice()
{
TestTraceListener trace = new TestTraceListener();
Log.AddListener(trace);
TestPlan target = getTestTestPlan();
target.Open();
target.Execute();
target.Execute();
target.Close();
Log.RemoveListener(trace);
trace.AssertErrors();
TestTestStep step = target.Steps[0] as TestTestStep;
Assert.AreEqual(2, step.PrePlanRunCount, "PrePlanRun was not called the correct number of times.");
Assert.AreEqual(2, step.PostPlanRunCount, "PostPlanRun was not called the correct number of times.");
}
public void RunCompositeMetaData2()
{
PlanRunCollectorListener listener = new PlanRunCollectorListener();
ResultSettings.Current.Add(listener);
TestTraceListener trace = new TestTraceListener();
Log.AddListener(trace);
TestPlan target = getTestTestPlan();
target.Open();
target.Execute();
target.Execute();
target.Close();
Log.RemoveListener(trace);
ResultSettings.Current.Remove(listener);
Assert.IsTrue(listener.PlanRuns.First().Parameters.Any(par => par.Value.ToString() == "Test Instrument"));
Assert.AreEqual(1, listener.PlanRuns.Last().Parameters.Count(par => par.Value.ToString() == "Test Instrument"));
}
public void RunCompositeAddInstrumentAfterOpen()
{
PlanRunCollectorListener listener = new PlanRunCollectorListener();
ResultSettings.Current.Add(listener);
TestTraceListener trace = new TestTraceListener();
Log.AddListener(trace);
TestPlan target = getTestTestPlan();
target.Open();
TestInstrument instr = InstrumentSettings.Current.FirstOrDefault(i => i is TestInstrument && (i as TestInstrument).Name.EndsWith("2")) as TestInstrument;
if (instr == null)
{
instr = new TestInstrument {
Name = "Test Instrument 2"
};
InstrumentSettings.Current.Add(instr);
}
(target.Steps[0] as TestTestStep).Instr = instr;
target.Execute();
listener.StepRuns.Clear();
target.Execute();
target.Close();
Log.RemoveListener(trace);
ResultSettings.Current.Remove(listener);
Assert.AreEqual(2, listener.PlanRuns.Select(run => run.StartTimeStamp).Distinct().Count());
Assert.AreEqual(1, listener.StepRuns.Count());
}
public void RunCompositeLogComparison()
{
// ## Fragile Test
// If the settings directory has been loaded in the meantime
// Log messages that new files are being created will appear on first run
// ComponentSettings are lazily loaded so that first happens when the test plan runs.
// We do one run to clear the state of the engine.
{
TestPlan target2 = getTestTestPlan();
Log.Flush();
target2.PrintTestPlanRunSummary = true;
target2.Execute();
Log.Flush();
}
TestTraceListener trace1 = new TestTraceListener();
Log.AddListener(trace1);
TestPlan target = getTestTestPlan();
target.PrintTestPlanRunSummary = true;
target.Execute();
Log.RemoveListener(trace1);
TestTraceListener trace2 = new TestTraceListener();
Log.AddListener(trace2);
target = getTestTestPlan();
target.PrintTestPlanRunSummary = true;
target.Open();
target.Execute();
target.Close();
Log.RemoveListener(trace2);
string allLog1 = trace1.allLog.ToString();
string allLog2 = trace2.allLog.ToString();
string[] log1Lines = filterLog(allLog1);
string[] log2Lines = filterLog(allLog2);
string[] log2LinesNoSpaces = filterLog(allLog2, true);
Assert.AreEqual(log1Lines.Count() + 2, log2Lines.Count(), allLog1 + Environment.NewLine + "##########" + Environment.NewLine + allLog2);
for (int i = 0; i < log1Lines.Length; i++)
{
var line = log1Lines[i].Replace(" ", "");
if (!log2LinesNoSpaces.Contains(line)) // We compare lines with removed spaces to avoid flakyness in CI.
{
// Print actual comparison data
Console.WriteLine($"Could not find '{line}' in following logs:");
foreach (var linez in log2LinesNoSpaces)
{
Console.WriteLine($"{linez}");
}
Console.WriteLine($"--------------- Printing logs without spaces removed ---------------");
// Print log data without their spaces removed
Console.WriteLine($"First run logs:");
foreach (var linez in log1Lines)
{
Console.WriteLine($"- {linez}");
}
Console.WriteLine($"Second run logs:");
foreach (var linez in log2Lines)
{
Console.WriteLine($"- {linez}");
}
Assert.Fail($"The logs from two testplan executions does not match...");
}
}
}