public static void TestCase(Assembly assembly,
RunMode mode,
int duration,
int threads,
int?exceptionThreshold = null,
bool printMethodName = false,
bool deadlockDetection = false,
int randomSeed = 0,
string[] selectedTests = null,
string[] overrides = null,
string[] variations = null,
string filter = null,
bool monitorEnabled = false,
string monitorMachineName = "localhost")
{
TestMetrics.Reset();
TestFinder.AssemblyName = assembly.GetName();
mode = RunMode.RunAll;
for (int i = 0; overrides != null && i < overrides.Length; i++)
{
TestMetrics.Overrides.Add(overrides[i], overrides[++i]);
}
for (int i = 0; variations != null && i < variations.Length; i++)
{
TestMetrics.Variations.Add(variations[i]);
}
for (int i = 0; selectedTests != null && i < selectedTests.Length; i++)
{
TestMetrics.SelectedTests.Add(selectedTests[i]);
}
TestMetrics.StressDuration = duration;
TestMetrics.StressThreads = threads;
TestMetrics.ExceptionThreshold = exceptionThreshold;
TestMetrics.MonitorEnabled = monitorEnabled;
TestMetrics.MonitorMachineName = monitorMachineName;
TestMetrics.RandomSeed = randomSeed;
TestMetrics.Filter = filter;
TestMetrics.PrintMethodName = printMethodName;
if (deadlockDetection)
{
DeadlockDetection.Enable();
}
// get and load all the tests
s_tests = TestFinder.GetTests(assembly);
// instantiate the stress engine
s_eng = new StressEngine(TestMetrics.StressThreads, TestMetrics.StressDuration, s_tests, TestMetrics.RandomSeed);
}
private void ExecuteTest(object targetInstance, TestMethodDelegate tmd)
{
int warmupIterations = _attr.WarmupIterations;
int testIterations = _attr.TestIterations;
if (_overrideIterations >= 0)
{
testIterations = _overrideIterations;
}
if (_overrideWarmup >= 0)
{
warmupIterations = _overrideWarmup;
}
/** do some cleanup to make memory tests more accurate **/
System.GC.Collect();
System.GC.WaitForPendingFinalizers();
System.GC.Collect();
IntPtr h = MemApi.GetCurrentProcess();
bool fRes = MemApi.SetProcessWorkingSetSize(h, -1, -1);
/****/
System.Threading.Thread.Sleep(10000);
for (int i = 0; i < warmupIterations; i++)
{
tmd(targetInstance);
}
TestMetrics.StartCollection();
for (int i = 0; i < testIterations; i++)
{
tmd(targetInstance);
}
TestMetrics.StopCollection();
}
public override void Run()
{
try
{
Stopwatch timer = new Stopwatch();
long warmupDuration = (long)_attr.WarmupDuration * Stopwatch.Frequency;
long testDuration = (long)_attr.TestDuration * Stopwatch.Frequency;
int threads = _attr.Threads;
TestInfo[] info = new TestInfo[threads];
ConstructorInfo targetConstructor = _type.GetConstructor(Type.EmptyTypes);
for (int i = 0; i < threads; i++)
{
info[i] = new TestInfo();
info[i]._instance = targetConstructor.Invoke(null);
info[i]._delegateTest = CreateTestMethodDelegate();
SetVariations(info[i]._instance);
ExecuteSetupPhase(info[i]._instance);
}
_firstException = null;
_continue = true;
_rps = 0;
for (int i = 0; i < threads; i++)
{
Interlocked.Increment(ref _threadsRunning);
Thread t = new Thread(new ParameterizedThreadStart(MultiThreadedTest.RunThread));
t.Start(info[i]);
}
timer.Reset();
timer.Start();
while (timer.ElapsedTicks < warmupDuration)
{
Thread.Sleep(1000);
}
int warmupRequests = Interlocked.Exchange(ref _rps, 0);
timer.Reset();
timer.Start();
TestMetrics.StartCollection();
while (timer.ElapsedTicks < testDuration)
{
Thread.Sleep(1000);
}
int requests = Interlocked.Exchange(ref _rps, 0);
double elapsedSeconds = timer.ElapsedTicks / Stopwatch.Frequency;
TestMetrics.StopCollection();
_continue = false;
while (_threadsRunning > 0)
{
Thread.Sleep(1000);
}
for (int i = 0; i < threads; i++)
{
ExecuteCleanupPhase(info[i]._instance);
}
double rps = (double)requests / elapsedSeconds;
if (_firstException == null)
{
LogTest(rps);
}
else
{
LogTestFailure(_firstException.ToString());
}
}
catch (TargetInvocationException e)
{
LogTestFailure(e.InnerException.ToString());
}
catch (Exception e)
{
LogTestFailure(e.ToString());
}
}
public static IEnumerable <TestBase> GetTests(Assembly assembly)
{
List <TestBase> tests = new List <TestBase>();
Type[] typesInModule = null;
try
{
typesInModule = assembly.GetTypes();
}
catch (ReflectionTypeLoadException ex)
{
Console.WriteLine("ReflectionTypeLoadException Errors");
foreach (Exception loadEx in ex.LoaderExceptions)
{
Console.WriteLine("\t" + loadEx.Message);
}
}
catch (Exception ex)
{
Console.WriteLine("Error." + ex.Message);
}
foreach (Type t in typesInModule)
{
MethodInfo[] methods = t.GetMethods(BindingFlags.Instance | BindingFlags.Public);
List <MethodInfo> setupMethods = new List <MethodInfo>();
List <MethodInfo> cleanupMethods = new List <MethodInfo>();
MethodInfo globalSetupMethod = null;
MethodInfo globalCleanupMethod = null;
MethodInfo globalExceptionHandlerMethod = null;
foreach (MethodInfo m in methods)
{
GlobalTestSetupAttribute[] globalSetupAttributes = (GlobalTestSetupAttribute[])m.GetCustomAttributes(typeof(GlobalTestSetupAttribute), true);
if (globalSetupAttributes.Length > 0)
{
if (null == globalSetupMethod)
{
globalSetupMethod = m;
}
else
{
throw new NotSupportedException("Only one GlobalTestSetup method may be specified per type.");
}
}
GlobalTestCleanupAttribute[] globalCleanupAttributes = (GlobalTestCleanupAttribute[])m.GetCustomAttributes(typeof(GlobalTestCleanupAttribute), true);
if (globalCleanupAttributes.Length > 0)
{
if (null == globalCleanupMethod)
{
globalCleanupMethod = m;
}
else
{
throw new NotSupportedException("Only one GlobalTestCleanup method may be specified per type.");
}
}
GlobalExceptionHandlerAttribute[] globalExceptionHandlerAttributes = (GlobalExceptionHandlerAttribute[])m.GetCustomAttributes(typeof(GlobalExceptionHandlerAttribute), true);
if (globalExceptionHandlerAttributes.Length > 0)
{
if (null == globalExceptionHandlerMethod)
{
globalExceptionHandlerMethod = m;
}
else
{
throw new NotSupportedException("Only one GlobalExceptionHandler method may be specified.");
}
}
TestSetupAttribute[] testSetupAttrs = (TestSetupAttribute[])m.GetCustomAttributes(typeof(TestSetupAttribute), true);
if (testSetupAttrs.Length > 0)
{
setupMethods.Add(m);;
}
TestCleanupAttribute[] testCleanupAttrs = (TestCleanupAttribute[])m.GetCustomAttributes(typeof(TestCleanupAttribute), true);
if (testCleanupAttrs.Length > 0)
{
cleanupMethods.Add(m);;
}
}
foreach (MethodInfo m in methods)
{
// add single-threaded tests to the list
TestAttribute[] testAttrs = (TestAttribute[])m.GetCustomAttributes(typeof(TestAttribute), true);
foreach (TestAttribute attr in testAttrs)
{
tests.Add(new Test(attr, m, t, setupMethods, cleanupMethods));
}
// add any declared stress tests.
StressTestAttribute[] stressTestAttrs = (StressTestAttribute[])m.GetCustomAttributes(typeof(StressTestAttribute), true);
foreach (StressTestAttribute attr in stressTestAttrs)
{
if (TestMetrics.IncludeTest(attr) && MatchFilter(attr))
{
tests.Add(new StressTest(attr, m, globalSetupMethod, globalCleanupMethod, t, setupMethods, cleanupMethods, globalExceptionHandlerMethod));
}
}
// add multi-threaded (non thread pool) tests to the list
MultiThreadedTestAttribute[] multiThreadedTestAttrs = (MultiThreadedTestAttribute[])m.GetCustomAttributes(typeof(MultiThreadedTestAttribute), true);
foreach (MultiThreadedTestAttribute attr in multiThreadedTestAttrs)
{
if (TestMetrics.IncludeTest(attr))
{
tests.Add(new MultiThreadedTest(attr, m, t, setupMethods, cleanupMethods));
}
}
// add multi-threaded (with thread pool) tests to the list
ThreadPoolTestAttribute[] threadPoolTestAttrs = (ThreadPoolTestAttribute[])m.GetCustomAttributes(typeof(ThreadPoolTestAttribute), true);
foreach (ThreadPoolTestAttribute attr in threadPoolTestAttrs)
{
if (TestMetrics.IncludeTest(attr))
{
tests.Add(new ThreadPoolTest(attr, m, t, setupMethods, cleanupMethods));
}
}
}
}
return(tests);
}