public static void TestCase(Assembly assembly, RunMode mode, int duration, int threads, int?exceptionThreshold = null, bool printMethodName = false, bool deadlockDetection = false, int randomSeed = 0, string[] selectedTests = null, string[] overrides = null, string[] variations = null, string filter = null, bool monitorEnabled = false, string monitorMachineName = "localhost") { TestMetrics.Reset(); TestFinder.AssemblyName = assembly.GetName(); mode = RunMode.RunAll; for (int i = 0; overrides != null && i < overrides.Length; i++) { TestMetrics.Overrides.Add(overrides[i], overrides[++i]); } for (int i = 0; variations != null && i < variations.Length; i++) { TestMetrics.Variations.Add(variations[i]); } for (int i = 0; selectedTests != null && i < selectedTests.Length; i++) { TestMetrics.SelectedTests.Add(selectedTests[i]); } TestMetrics.StressDuration = duration; TestMetrics.StressThreads = threads; TestMetrics.ExceptionThreshold = exceptionThreshold; TestMetrics.MonitorEnabled = monitorEnabled; TestMetrics.MonitorMachineName = monitorMachineName; TestMetrics.RandomSeed = randomSeed; TestMetrics.Filter = filter; TestMetrics.PrintMethodName = printMethodName; if (deadlockDetection) { DeadlockDetection.Enable(); } // get and load all the tests s_tests = TestFinder.GetTests(assembly); // instantiate the stress engine s_eng = new StressEngine(TestMetrics.StressThreads, TestMetrics.StressDuration, s_tests, TestMetrics.RandomSeed); }
private void ExecuteTest(object targetInstance, TestMethodDelegate tmd) { int warmupIterations = _attr.WarmupIterations; int testIterations = _attr.TestIterations; if (_overrideIterations >= 0) { testIterations = _overrideIterations; } if (_overrideWarmup >= 0) { warmupIterations = _overrideWarmup; } /** do some cleanup to make memory tests more accurate **/ System.GC.Collect(); System.GC.WaitForPendingFinalizers(); System.GC.Collect(); IntPtr h = MemApi.GetCurrentProcess(); bool fRes = MemApi.SetProcessWorkingSetSize(h, -1, -1); /****/ System.Threading.Thread.Sleep(10000); for (int i = 0; i < warmupIterations; i++) { tmd(targetInstance); } TestMetrics.StartCollection(); for (int i = 0; i < testIterations; i++) { tmd(targetInstance); } TestMetrics.StopCollection(); }
public override void Run() { try { Stopwatch timer = new Stopwatch(); long warmupDuration = (long)_attr.WarmupDuration * Stopwatch.Frequency; long testDuration = (long)_attr.TestDuration * Stopwatch.Frequency; int threads = _attr.Threads; TestInfo[] info = new TestInfo[threads]; ConstructorInfo targetConstructor = _type.GetConstructor(Type.EmptyTypes); for (int i = 0; i < threads; i++) { info[i] = new TestInfo(); info[i]._instance = targetConstructor.Invoke(null); info[i]._delegateTest = CreateTestMethodDelegate(); SetVariations(info[i]._instance); ExecuteSetupPhase(info[i]._instance); } _firstException = null; _continue = true; _rps = 0; for (int i = 0; i < threads; i++) { Interlocked.Increment(ref _threadsRunning); Thread t = new Thread(new ParameterizedThreadStart(MultiThreadedTest.RunThread)); t.Start(info[i]); } timer.Reset(); timer.Start(); while (timer.ElapsedTicks < warmupDuration) { Thread.Sleep(1000); } int warmupRequests = Interlocked.Exchange(ref _rps, 0); timer.Reset(); timer.Start(); TestMetrics.StartCollection(); while (timer.ElapsedTicks < testDuration) { Thread.Sleep(1000); } int requests = Interlocked.Exchange(ref _rps, 0); double elapsedSeconds = timer.ElapsedTicks / Stopwatch.Frequency; TestMetrics.StopCollection(); _continue = false; while (_threadsRunning > 0) { Thread.Sleep(1000); } for (int i = 0; i < threads; i++) { ExecuteCleanupPhase(info[i]._instance); } double rps = (double)requests / elapsedSeconds; if (_firstException == null) { LogTest(rps); } else { LogTestFailure(_firstException.ToString()); } } catch (TargetInvocationException e) { LogTestFailure(e.InnerException.ToString()); } catch (Exception e) { LogTestFailure(e.ToString()); } }
public static IEnumerable <TestBase> GetTests(Assembly assembly) { List <TestBase> tests = new List <TestBase>(); Type[] typesInModule = null; try { typesInModule = assembly.GetTypes(); } catch (ReflectionTypeLoadException ex) { Console.WriteLine("ReflectionTypeLoadException Errors"); foreach (Exception loadEx in ex.LoaderExceptions) { Console.WriteLine("\t" + loadEx.Message); } } catch (Exception ex) { Console.WriteLine("Error." + ex.Message); } foreach (Type t in typesInModule) { MethodInfo[] methods = t.GetMethods(BindingFlags.Instance | BindingFlags.Public); List <MethodInfo> setupMethods = new List <MethodInfo>(); List <MethodInfo> cleanupMethods = new List <MethodInfo>(); MethodInfo globalSetupMethod = null; MethodInfo globalCleanupMethod = null; MethodInfo globalExceptionHandlerMethod = null; foreach (MethodInfo m in methods) { GlobalTestSetupAttribute[] globalSetupAttributes = (GlobalTestSetupAttribute[])m.GetCustomAttributes(typeof(GlobalTestSetupAttribute), true); if (globalSetupAttributes.Length > 0) { if (null == globalSetupMethod) { globalSetupMethod = m; } else { throw new NotSupportedException("Only one GlobalTestSetup method may be specified per type."); } } GlobalTestCleanupAttribute[] globalCleanupAttributes = (GlobalTestCleanupAttribute[])m.GetCustomAttributes(typeof(GlobalTestCleanupAttribute), true); if (globalCleanupAttributes.Length > 0) { if (null == globalCleanupMethod) { globalCleanupMethod = m; } else { throw new NotSupportedException("Only one GlobalTestCleanup method may be specified per type."); } } GlobalExceptionHandlerAttribute[] globalExceptionHandlerAttributes = (GlobalExceptionHandlerAttribute[])m.GetCustomAttributes(typeof(GlobalExceptionHandlerAttribute), true); if (globalExceptionHandlerAttributes.Length > 0) { if (null == globalExceptionHandlerMethod) { globalExceptionHandlerMethod = m; } else { throw new NotSupportedException("Only one GlobalExceptionHandler method may be specified."); } } TestSetupAttribute[] testSetupAttrs = (TestSetupAttribute[])m.GetCustomAttributes(typeof(TestSetupAttribute), true); if (testSetupAttrs.Length > 0) { setupMethods.Add(m);; } TestCleanupAttribute[] testCleanupAttrs = (TestCleanupAttribute[])m.GetCustomAttributes(typeof(TestCleanupAttribute), true); if (testCleanupAttrs.Length > 0) { cleanupMethods.Add(m);; } } foreach (MethodInfo m in methods) { // add single-threaded tests to the list TestAttribute[] testAttrs = (TestAttribute[])m.GetCustomAttributes(typeof(TestAttribute), true); foreach (TestAttribute attr in testAttrs) { tests.Add(new Test(attr, m, t, setupMethods, cleanupMethods)); } // add any declared stress tests. StressTestAttribute[] stressTestAttrs = (StressTestAttribute[])m.GetCustomAttributes(typeof(StressTestAttribute), true); foreach (StressTestAttribute attr in stressTestAttrs) { if (TestMetrics.IncludeTest(attr) && MatchFilter(attr)) { tests.Add(new StressTest(attr, m, globalSetupMethod, globalCleanupMethod, t, setupMethods, cleanupMethods, globalExceptionHandlerMethod)); } } // add multi-threaded (non thread pool) tests to the list MultiThreadedTestAttribute[] multiThreadedTestAttrs = (MultiThreadedTestAttribute[])m.GetCustomAttributes(typeof(MultiThreadedTestAttribute), true); foreach (MultiThreadedTestAttribute attr in multiThreadedTestAttrs) { if (TestMetrics.IncludeTest(attr)) { tests.Add(new MultiThreadedTest(attr, m, t, setupMethods, cleanupMethods)); } } // add multi-threaded (with thread pool) tests to the list ThreadPoolTestAttribute[] threadPoolTestAttrs = (ThreadPoolTestAttribute[])m.GetCustomAttributes(typeof(ThreadPoolTestAttribute), true); foreach (ThreadPoolTestAttribute attr in threadPoolTestAttrs) { if (TestMetrics.IncludeTest(attr)) { tests.Add(new ThreadPoolTest(attr, m, t, setupMethods, cleanupMethods)); } } } } return(tests); }