bool IsSelectedTest(ParallelTest test, List <string> testsList) { if (testsList == null) { return(true); } return(testsList.Contains(test.Name)); }
internal static TestConf GetTestConfFromName( ParallelTest testGroup, string testName) { foreach (TestConf testConf in testGroup.Tests) { if (testConf.Name == testName) { return(testConf); } } return(null); }
internal PNUnitService( ParallelTest testGroup, LogWriter logWriter, TestsRun testsRun, LauncherStatus launcherStatus) { mTestGroup = testGroup; mLogWriter = logWriter; mTestsRun = testsRun; mbBarriersInitialized = false; mLauncherStatus = launcherStatus; mFinish = new ManualResetEvent(false); }
private static void ReplaceVariableOnTest(ParallelTest t, Variable var) { t.Agents = ReplaceVarArray(t.Agents, var); t.Name = ReplaceVar(t.Name, var); foreach (TestConf tc in t.Tests) { tc.Name = ReplaceVar(tc.Name, var); tc.Assembly = ReplaceVar(tc.Assembly, var); tc.TestToRun = ReplaceVar(tc.TestToRun, var); tc.Machine = ReplaceVar(tc.Machine, var); tc.TestParams = ReplaceVarArray(tc.TestParams, var); tc.StartBarrier = ReplaceVar(tc.StartBarrier, var); tc.EndBarrier = ReplaceVar(tc.EndBarrier, var); tc.WaitBarriers = ReplaceVarArray(tc.WaitBarriers, var); } }
internal Runner( ParallelTest test, Hashtable userValues, int testTimeout, LogWriter logWriter, LauncherStatus launcherStatus, string listenAddress) { mTestGroup = test; mUserValues = userValues; mTestTimeout = testTimeout; mLogWriter = logWriter; mStatus = launcherStatus; mListenAddress = listenAddress; mTestsRun = new TestsRun(); }
public Runner(ParallelTest test, Hashtable userValues) { mTestGroup = test; mResults = new ArrayList(); mUserValues = userValues; }
// Variable replacement private static void ReplaceVariables(ParallelTest[] target, Variable[] variables) { if (variables == null || target == null) return; foreach (Variable var in variables) foreach (ParallelTest test in target) ReplaceVariableOnTest(test, var); }
private static void ReplaceVariableOnTest(ParallelTest t, Variable var) { t.Agents = ReplaceVarArray(t.Agents, var); t.Name = ReplaceVar (t.Name, var); foreach (TestConf tc in t.Tests) { tc.Name = ReplaceVar(tc.Name, var); tc.Assembly = ReplaceVar(tc.Assembly, var); tc.TestToRun = ReplaceVar(tc.TestToRun, var); tc.Machine = ReplaceVar(tc.Machine, var); tc.TestParams = ReplaceVarArray(tc.TestParams, var); tc.StartBarrier = ReplaceVar(tc.StartBarrier, var); tc.EndBarrier = ReplaceVar(tc.EndBarrier, var); tc.WaitBarriers = ReplaceVarArray(tc.WaitBarriers, var); } }
public Runner(ParallelTest test) { mTestGroup = test; mResults = new ArrayList(); }
internal Runner[] RunTests( TestGroup group, List <string> testsList, int maxRetry, bool bShellMode, int retryOnFailure, string failedfile, TestSuiteLogger testSuiteLogger, int testsTimeout, TestRange testRange, Hashtable userValues, LogWriter logWriter, string listenAddress, string reportFile) { if (testRange == null) { mLog.Warn("No tests are selected to run. Exiting."); return(new Runner[0]); } int testCount = testRange.EndTest - testRange.StartTest + 1; int testToExecuteCount = (testsList != null) ? testsList.Count : testCount; mStatus = new LauncherStatus(testCount, testToExecuteCount); Runner[] runners = new Runner[testCount]; List <ParallelTest> failedGroups = new List <ParallelTest>(); for (int currentTest = testRange.StartTest; currentTest <= testRange.EndTest;) { int ini = Environment.TickCount; ParallelTest test = group.ParallelTests[currentTest] as ParallelTest; if (!IsSelectedTest(test, testsList)) { mLog.ErrorFormat( "Test with name [{0}] is not invited to this party.", test.Name); ++currentTest; mStatus.Increment(); continue; } mStatus.SetCurrentTestName(test.Name); int retryCount = 0; bool bRetry = true; while (bRetry && retryCount < maxRetry) { bRetry = false; LogTestProgress(group, testRange, testCount, currentTest); Runner runner = new Runner( test, userValues, testsTimeout, logWriter, mStatus, listenAddress); if (bShellMode) { runner.ShellMode = bShellMode; } runners[currentTest - testRange.StartTest] = runner; if (reportFile != null) { StatusReport.Write(reportFile, mStatus, false); } runner.Run(); TestResult[] runnerResults = runner.GetTestResults(); if (runnerResults == null) { mLog.InfoFormat("Error. Results for test [{0}] are NULL", test.Name); ++currentTest; mStatus.Increment(); mStatus.IncrementExecuted(); continue; } bool isRepeated = retryCount > 0; if (reportFile == null) { LogTestResultsToTTS( testSuiteLogger, runnerResults, test.Name, isRepeated); } else { LogTestResultsToFile( reportFile, runnerResults, test.Name, isRepeated, true); } bRetry = RetryTest(runnerResults); bool bFailed = FailedTest(runnerResults); if (bRetry || ((bFailed && (retryOnFailure > 0) && ((retryCount + 1) < maxRetry)) /* so that list time is printed*/)) { bRetry = true; ++retryCount; mLog.Info("Test failed with retry option, trying again"); mStatus.AddRepeated(test.Name); continue; } if (bFailed) { failedGroups.Add(test); WriteGroup(failedGroups, failedfile); mStatus.AddFailed(test.Name); } if (IgnoredTest(runnerResults)) { mStatus.AddIgnored(test.Name); } } // updated at the bottom so it's not affected by retries mStatus.Increment(); mStatus.IncrementExecuted(); ++currentTest; mLog.DebugFormat("Test {0} time {1} ms", test.Name, Environment.TickCount - ini); } if (reportFile != null) { StatusReport.Write(reportFile, mStatus, true); } return(runners); }
static void Main(string[] args) { string resultfile = null; string failedfile = null; string passedfile = null; int retryOnFailure = 0; int maxRetry = MAX_TEST_RETRY; ConfigureLogging(); try { // Load the test configuration file if (args.Length == 0) { Console.WriteLine( "Usage: launcher configfile [--result=filename] [--failed=filename] [-D:var=value] [-val:variable=value] [--retry=number] [--range=from-to] [--test=testname]"); return; } string configfile = args[0]; mTestPath = Path.GetDirectoryName(configfile); TestGroup group = TestConfLoader.LoadFromFile(configfile, args); int startTest = 0; int endTest = group.ParallelTests.Length - 1; failedfile = Path.Combine(mTestPath, "smokefailed.conf"); passedfile = Path.Combine(mTestPath, "smokepassed.conf"); if (args.Length > 1) { foreach (string arg in args) { if (arg.StartsWith("--result=")) { resultfile = arg.Substring(9); resultfile = Path.GetFullPath(resultfile); } if (arg.StartsWith("--failed=")) { failedfile = arg.Substring(9); failedfile = Path.GetFullPath(failedfile); } if (arg.StartsWith("--retry=")) { retryOnFailure = int.Parse(arg.Substring("--retry=".Length)); log.InfoFormat("Retry on failure activated. {0} retries", retryOnFailure); maxRetry = retryOnFailure; } if (arg.StartsWith("--test=")) { string testName = arg.Substring("--test=".Length); int index = -1; for (int i = 0; i < group.ParallelTests.Length; i++) { if (group.ParallelTests[i].Name != testName) { continue; } index = i; break; } if (index == -1) { Console.WriteLine("The specified test was not found"); return; } startTest = index; endTest = index; } if (arg.StartsWith("--range=")) { string range = arg.Substring("--range=".Length); // now range should be something like xx-xx if (range.IndexOf("-") < 0) { Console.WriteLine("Test range incorrectly specified, it must be something like 0-10"); return; } string[] ranges = range.Split('-'); if (ranges.Length != 2) { Console.WriteLine("Test range incorrectly specified, it must be something like 0-10"); return; } startTest = int.Parse(ranges[0]); endTest = int.Parse(ranges[1]); if ((startTest > endTest) || (startTest < 0) || (startTest > group.ParallelTests.Length - 1)) { Console.WriteLine("Start test must be in a correct test range"); return; } if ((endTest < startTest) || (endTest < 0) || (endTest > group.ParallelTests.Length - 1)) { Console.WriteLine("End test must be in a correct test range"); return; } log.InfoFormat("Starting test range [{0}-{1}]", startTest, endTest); } } } if ((group == null) || (group.ParallelTests.Length == 0)) { Console.WriteLine("No tests to run"); return; } Hashtable userValues = GetUserValues(args); ConfigureRemoting(); ArrayList failedGroups = new ArrayList(); ArrayList passedGroups = new ArrayList(); int testCount = endTest - startTest + 1; // Each parallel test is launched sequencially... Runner[] runners = new Runner[testCount]; DateTime beginTimestamp = DateTime.Now; for (int i = startTest; i <= endTest;) { ParallelTest test = group.ParallelTests[i] as ParallelTest; int retryCount = 0; bool bRetry = true; while (bRetry && retryCount < maxRetry) { bRetry = false; if (testCount != group.ParallelTests.Length) { log.InfoFormat("Test {0} of {1}. {2}/{3}", i, group.ParallelTests.Length, i - startTest + 1, testCount); } else { log.InfoFormat("Test {0} of {1}", i + 1, group.ParallelTests.Length); } Runner runner = new Runner(test, userValues); runner.Run(); runners[i - startTest] = runner; // Wait to finish runner.Join(); TestResult[] runnerResults = runner.GetTestResults(); if (runnerResults == null) { log.Info("Error. Results are NULL"); ++i; continue; } bRetry = RetryTest(runnerResults); bool bFailed = FailedTest(runnerResults); if (bRetry || ((bFailed && (retryOnFailure > 0) && ((retryCount + 1) < maxRetry)) /* so that list time is printed*/)) { bRetry = true; ++retryCount; log.Info("Test failed with retry option, trying again"); continue; } if (bFailed) { failedGroups.Add(test); WriteGroup(failedGroups, failedfile); } else { passedGroups.Add(test); WriteGroup(passedGroups, passedfile); } } // updated at the bottom so it's not affected by retries ++i; } DateTime endTimestamp = DateTime.Now; // Print the results double TotalBiggerTime = 0; int TotalTests = 0; int TotalExecutedTests = 0; int TotalFailedTests = 0; int TotalSuccessTests = 0; IList failedTests = new ArrayList(); int j; foreach (Runner runner in runners) { int ExecutedTests = 0; int FailedTests = 0; int SuccessTests = 0; double BiggerTime = 0; TestResult[] results = runner.GetTestResults(); Log(string.Format("==== Tests Results for Parallel TestGroup {0} ===", runner.TestGroupName)); j = 0; foreach (TestResult res in results) { if (res.Executed) { ++ExecutedTests; } if (res.IsFailure) { ++FailedTests; } if (res.IsSuccess) { ++SuccessTests; } PrintResult(++j, res); if (res.Time > BiggerTime) { BiggerTime = res.Time; } if (res.IsFailure) { failedTests.Add(res); } } Log("Summary:"); Log(string.Format("\tTotal: {0}\r\n\tExecuted: {1}\r\n\tFailed: {2}\r\n\tSuccess: {3}\r\n\t% Success: {4}\r\n\tBiggest Execution Time: {5} s\r\n", results.Length, ExecutedTests, FailedTests, SuccessTests, results.Length > 0 ? 100 * SuccessTests / results.Length : 0, BiggerTime)); TotalTests += results.Length; TotalExecutedTests += ExecutedTests; TotalFailedTests += FailedTests; TotalSuccessTests += SuccessTests; TotalBiggerTime += BiggerTime; } // print all failed tests together if (failedTests.Count > 0) { Log("==== Failed tests ==="); for (j = 0; j < failedTests.Count; ++j) { PrintResult(j, failedTests[j] as PNUnitTestResult); } } if (runners.Length > 1) { Log("Summary for all the parallel tests:"); Log(string.Format("\tTotal: {0}\r\n\tExecuted: {1}\r\n\tFailed: {2}\r\n\tSuccess: {3}\r\n\t% Success: {4}\r\n\tBiggest Execution Time: {5} s\r\n", TotalTests, TotalExecutedTests, TotalFailedTests, TotalSuccessTests, TotalTests > 0 ? 100 * TotalSuccessTests / TotalTests : 0, TotalBiggerTime)); } TimeSpan elapsedTime = endTimestamp.Subtract(beginTimestamp); Log(string.Format("Launcher execution time: {0} seconds", elapsedTime.TotalSeconds)); } finally { WriteResult(resultfile); } }