private static Type FindImpl()
{
Platform.OS os = Platform.GetOS();
Type type = typeof(T);
var cls = ClassLoader.FindTypeByAttribute <PlatformImplAttribute>((t, a) => a.OS == os && (t.BaseType == type || t.GetInterfaces().Contains(type)));
if (cls == null)
{
throw new TypeLoadException("Could not find an instance of '" + type.FullName + "' for the " + os + " platform.");
}
return(cls);
}
private static T LoadInstance()
{
Platform.OS os = Platform.GetOS();
Type type = typeof(T);
var cls = ClassLoader.FindTypeByAttribute <PlatformImplAttribute>((t, a) => a.OS == os && t.GetInterfaces().Contains(type));
if (cls == null)
{
throw new TypeLoadException("Could not find an instance of '" + type.FullName + "' for the " + os + " platform.");
}
object obj = Activator.CreateInstance(cls);
T ret = obj as T;
return(ret);
}
/// <summary>
/// Run a test case. Contains main fuzzing loop.
/// </summary>
/// <param name="dom"></param>
/// <param name="test"></param>
/// <param name="context"></param>
protected void runTest(Dom.Dom dom, Test test, RunContext context)
{
try
{
context.test = test;
context.test.strategy.Context = context;
context.test.strategy.Engine = this;
context.agentManager = new AgentManager(context);
context.reproducingFault = false;
context.reproducingIterationJumpCount = 1;
if (context.config.userDefinedSeed && !test.strategy.UsesRandomSeed)
{
var attr = ClassLoader.GetAttributes <MutationStrategyAttribute>(test.strategy.GetType(), null).Where(a => a.IsDefault == true).FirstOrDefault();
var name = attr != null ? attr.Name : test.strategy.GetType().Name;
var msg = "The '{0}' mutation strategy does not allow setting the random seed.".Fmt(name);
OnTestWarning(context, msg);
}
// Get mutation strategy
MutationStrategy mutationStrategy = test.strategy;
mutationStrategy.Initialize(context, this);
uint iterationStart = 1;
uint iterationStop = uint.MaxValue;
uint?iterationTotal = null;
uint lastControlIteration = 0;
uint redoCount = 0;
if (!mutationStrategy.IsDeterministic)
{
if (context.config.parallel)
{
throw new PeachException("parallel is not supported when a non-deterministic mutation strategy is used");
}
if (context.config.countOnly)
{
throw new PeachException("count is not supported when a non-deterministic mutation strategy is used");
}
}
if (context.config.range)
{
if (context.config.parallel)
{
throw new PeachException("range is not supported when parallel is used");
}
logger.Debug("runTest: context.config.range == true, start: " +
context.config.rangeStart + ", stop: " + context.config.rangeStop);
iterationStart = context.config.rangeStart;
iterationStop = context.config.rangeStop;
}
else if (context.config.skipToIteration > 1)
{
logger.Debug("runTest: context.config.skipToIteration == " +
context.config.skipToIteration);
iterationStart = context.config.skipToIteration;
}
iterationStart = Math.Max(1, iterationStart);
uint lastReproFault = iterationStart - 1;
uint iterationCount = iterationStart;
bool firstRun = true;
// First iteration is always a control/recording iteration
context.controlIteration = true;
context.controlRecordingIteration = true;
test.markMutableElements();
OnTestStarting(context);
// Start agents
foreach (Dom.Agent agent in test.agents.Values)
{
// Only use agent if on correct platform
if ((agent.platform & Platform.GetOS()) != Platform.OS.None)
{
context.agentManager.AgentConnect(agent);
context.agentManager.GetAgent(agent.name).SessionStarting();
// Note: We want to perfrom SessionStarting on each agent
// in turn. We do this incase the first agent starts
// a virtual machine that contains the second agent.
}
}
while ((firstRun || iterationCount <= iterationStop) && context.continueFuzzing)
{
context.currentIteration = iterationCount;
firstRun = false;
// Clear out or iteration based state store
context.iterationStateStore.Clear();
// Should we perform a control iteration?
if (test.controlIteration > 0 && !context.reproducingFault)
{
if ((test.controlIteration == 1 || iterationCount % test.controlIteration == 1) && lastControlIteration != iterationCount)
{
context.controlIteration = true;
}
}
try
{
// Must set iteration 1st as strategy could enable control/record bools
mutationStrategy.Iteration = iterationCount;
if (context.controlIteration && context.controlRecordingIteration)
{
context.controlRecordingActionsExecuted.Clear();
context.controlRecordingStatesExecuted.Clear();
}
context.controlActionsExecuted.Clear();
context.controlStatesExecuted.Clear();
if (context.config.singleIteration && !context.controlIteration && iterationCount == 1)
{
logger.Debug("runTest: context.config.singleIteration == true");
break;
}
// Make sure we are not hanging on to old faults.
context.faults.Clear();
try
{
if (IterationStarting != null)
{
IterationStarting(context, iterationCount, iterationTotal.HasValue ? iterationStop : iterationTotal);
}
if (context.controlIteration)
{
if (context.controlRecordingIteration)
{
logger.Debug("runTest: Performing recording iteration.");
}
else
{
logger.Debug("runTest: Performing control iteration.");
}
}
context.agentManager.IterationStarting(iterationCount, context.reproducingFault);
test.stateModel.Run(context);
}
catch (SoftException se)
{
// We should just eat SoftExceptions.
// They indicate we should move to the next
// iteration.
if (context.controlRecordingIteration)
{
logger.Debug("runTest: SoftException on control recording iteration");
if (se.InnerException != null && string.IsNullOrEmpty(se.Message))
{
throw new PeachException(se.InnerException.Message, se);
}
throw new PeachException(se.Message, se);
}
if (context.controlIteration)
{
logger.Debug("runTest: SoftException on control iteration, saving as fault");
var ex = se.InnerException ?? se;
OnControlFault(context, iterationCount, "SoftException Detected:\n" + ex.ToString());
}
logger.Debug("runTest: SoftException, skipping to next iteration");
}
catch (PathException)
{
// We should just eat PathException.
// They indicate we should move to the next
// iteration.
logger.Debug("runTest: PathException, skipping to next iteration");
}
catch (System.OutOfMemoryException ex)
{
logger.Debug(ex.Message);
logger.Debug(ex.StackTrace);
logger.Debug("runTest: " +
"Warning: Iteration ended due to out of memory exception. Continuing to next iteration.");
throw new SoftException("Out of memory");
}
finally
{
context.agentManager.IterationFinished();
if (IterationFinished != null)
{
IterationFinished(context, iterationCount);
}
// If this was a control iteration, verify it againt our origional
// recording.
if (context.controlRecordingIteration == false &&
context.controlIteration &&
!test.nonDeterministicActions)
{
if (context.controlRecordingActionsExecuted.Count != context.controlActionsExecuted.Count)
{
string description = string.Format(@"The Peach control iteration performed failed
to execute same as initial control. Number of actions is different. {0} != {1}",
context.controlRecordingActionsExecuted.Count,
context.controlActionsExecuted.Count);
logger.Debug(description);
OnControlFault(context, iterationCount, description);
}
else if (context.controlRecordingStatesExecuted.Count != context.controlStatesExecuted.Count)
{
string description = string.Format(@"The Peach control iteration performed failed
to execute same as initial control. Number of states is different. {0} != {1}",
context.controlRecordingStatesExecuted.Count,
context.controlStatesExecuted.Count);
logger.Debug(description);
OnControlFault(context, iterationCount, description);
}
if (context.faults.Count == 0)
{
foreach (Dom.Action action in context.controlRecordingActionsExecuted)
{
if (!context.controlActionsExecuted.Contains(action))
{
string description = @"The Peach control iteration performed failed
to execute same as initial control. Action " + action.name + " was not performed.";
logger.Debug(description);
OnControlFault(context, iterationCount, description);
}
}
}
if (context.faults.Count == 0)
{
foreach (Dom.State state in context.controlRecordingStatesExecuted)
{
if (!context.controlStatesExecuted.Contains(state))
{
string description = @"The Peach control iteration performed failed
to execute same as initial control. State " + state.name + "was not performed.";
logger.Debug(description);
OnControlFault(context, iterationCount, description);
}
}
}
}
}
// User can specify a time to wait between iterations
// we can use that time to better detect faults
if (context.test.waitTime > 0)
{
Thread.Sleep((int)(context.test.waitTime * 1000));
}
if (context.reproducingFault)
{
// User can specify a time to wait between iterations
// when reproducing faults.
if (context.test.faultWaitTime > 0)
{
Thread.Sleep((int)(context.test.faultWaitTime * 1000));
}
}
// Collect any faults that were found
context.OnCollectFaults();
if (context.faults.Count > 0)
{
logger.Debug("runTest: detected fault on iteration " + iterationCount);
foreach (Fault fault in context.faults)
{
fault.iteration = iterationCount;
fault.controlIteration = context.controlIteration;
fault.controlRecordingIteration = context.controlRecordingIteration;
}
if (context.reproducingFault || !test.replayEnabled)
{
OnFault(context, iterationCount, test.stateModel, context.faults.ToArray());
}
else
{
OnReproFault(context, iterationCount, test.stateModel, context.faults.ToArray());
}
if (context.controlRecordingIteration && (!test.replayEnabled || context.reproducingFault))
{
logger.Debug("runTest: Fault detected on control iteration");
throw new PeachException("Fault detected on control iteration.");
}
if (context.reproducingFault)
{
lastReproFault = iterationCount;
// If we have moved less than 20 iterations, start fuzzing
// from here thinking we may have not really performed the
// next few iterations.
// Otherwise skip forward to were we left off.
if (context.reproducingInitialIteration - iterationCount > 20)
{
iterationCount = (uint)context.reproducingInitialIteration;
}
context.reproducingFault = false;
context.reproducingIterationJumpCount = 1;
logger.Debug("runTest: Reproduced fault, continuing fuzzing at iteration " + iterationCount);
}
else if (test.replayEnabled)
{
logger.Debug("runTest: Attempting to reproduce fault.");
context.reproducingFault = true;
context.reproducingInitialIteration = iterationCount;
context.reproducingIterationJumpCount = 1;
// User can specify a time to wait between iterations
// we can use that time to better detect faults
if (context.test.waitTime > 0)
{
Thread.Sleep((int)(context.test.waitTime * 1000));
}
// User can specify a time to wait between iterations
// when reproducing faults.
if (context.test.faultWaitTime > 0)
{
Thread.Sleep((int)(context.test.faultWaitTime * 1000));
}
logger.Debug("runTest: replaying iteration " + iterationCount);
continue;
}
}
else if (context.reproducingFault)
{
uint maxJump = context.reproducingInitialIteration - lastReproFault - 1;
if (context.reproducingIterationJumpCount >= (maxJump * 2) || context.reproducingIterationJumpCount > context.reproducingMaxBacksearch)
{
logger.Debug("runTest: Giving up reproducing fault, reached max backsearch.");
context.reproducingFault = false;
iterationCount = context.reproducingInitialIteration;
OnReproFailed(context, iterationCount);
}
else
{
uint delta = Math.Min(maxJump, context.reproducingIterationJumpCount);
iterationCount = (uint)context.reproducingInitialIteration - delta - 1;
logger.Debug("runTest: " +
"Moving backwards " + delta + " iterations to reproduce fault.");
}
// Make next jump larger
context.reproducingIterationJumpCount *= context.reproducingSkipMultiple;
}
if (context.agentManager.MustStop())
{
logger.Debug("runTest: agents say we must stop!");
throw new PeachException("Error, agent monitor stopped run!");
}
// Update our totals and stop based on new count
if (context.controlIteration && context.controlRecordingIteration && !iterationTotal.HasValue)
{
if (context.config.countOnly)
{
OnHaveCount(context, mutationStrategy.Count);
break;
}
iterationTotal = mutationStrategy.Count;
if (iterationTotal < iterationStop)
{
iterationStop = iterationTotal.Value;
}
if (context.config.parallel)
{
if (iterationTotal < context.config.parallelTotal)
{
throw new PeachException(string.Format("Error, {1} parallel machines is greater than the {0} total iterations.", iterationTotal, context.config.parallelTotal));
}
var range = Utilities.SliceRange(1, iterationStop, context.config.parallelNum, context.config.parallelTotal);
iterationStart = range.Item1;
iterationStop = range.Item2;
OnHaveParallel(context, iterationStart, iterationStop);
if (context.config.skipToIteration > iterationStart)
{
iterationStart = context.config.skipToIteration;
}
iterationCount = iterationStart;
}
}
// Don't increment the iteration count if we are on a
// control iteration
if (!context.controlIteration)
{
++iterationCount;
}
redoCount = 0;
}
catch (RedoIterationException rte)
{
logger.Debug("runTest: redoing test iteration for the " + redoCount + " time.");
// Repeat the same iteration unless
// we have already retried 3 times.
if (redoCount >= 3)
{
throw new PeachException(rte.Message, rte);
}
redoCount++;
}
finally
{
if (!context.reproducingFault)
{
if (context.controlIteration)
{
lastControlIteration = iterationCount;
}
context.controlIteration = false;
context.controlRecordingIteration = false;
}
}
}
}
catch (MutatorCompleted)
{
// Ignore, signals end of fuzzing run
logger.Debug("runTest: MutatorCompleted exception, ending fuzzing");
}
finally
{
foreach (Publisher publisher in context.test.publishers.Values)
{
try
{
publisher.stop();
}
catch
{
}
}
context.agentManager.SessionFinished();
context.agentManager.StopAllMonitors();
context.agentManager.Shutdown();
OnTestFinished(context);
context.test = null;
test.strategy.Finalize(context, this);
}
}
public static void Print()
{
var color = Console.ForegroundColor;
var domTypes = new SortedDictionary<string, Type>();
foreach (var type in ClassLoader.GetAllByAttribute<Peach.Core.Dom.DataElementAttribute>(null))
{
if (domTypes.ContainsKey(type.Key.elementName))
{
PrintDuplicate("Data element", type.Key.elementName, domTypes[type.Key.elementName], type.Value);
continue;
}
domTypes.Add(type.Key.elementName, type.Value);
}
var pluginsByName = new SortedDictionary<string, Type>();
var plugins = new SortedDictionary<Type, SortedDictionary<Type, SortedSet<PluginAttribute>>>(new TypeComparer());
foreach (var type in ClassLoader.GetAllByAttribute<Peach.Core.PluginAttribute>(null))
{
if (type.Key.IsTest)
continue;
var pluginType = type.Key.Type;
string fullName = type.Key.Type.Name + ": " + type.Key.Name;
if (pluginsByName.ContainsKey(fullName))
{
PrintDuplicate(type.Key.Type.Name, type.Key.Name, pluginsByName[fullName], type.Value);
continue;
}
pluginsByName.Add(fullName, type.Value);
if (!plugins.ContainsKey(pluginType))
plugins.Add(pluginType, new SortedDictionary<Type, SortedSet<PluginAttribute>>(new TypeComparer()));
var plugin = plugins[pluginType];
if (!plugin.ContainsKey(type.Value))
plugin.Add(type.Value, new SortedSet<PluginAttribute>(new PluginComparer()));
var attrs = plugin[type.Value];
bool added = attrs.Add(type.Key);
System.Diagnostics.Debug.Assert(added);
}
Console.WriteLine("----- Data Elements --------------------------------------------");
foreach (var elem in domTypes)
{
Console.WriteLine();
Console.WriteLine(" {0}", elem.Key);
PrintParams(elem.Value);
}
foreach (var kv in plugins)
{
Console.WriteLine();
Console.WriteLine();
Console.WriteLine("----- {0}s --------------------------------------------", kv.Key.Name);
foreach (var plugin in kv.Value)
{
Console.WriteLine();
Console.Write(" ");
foreach (var attr in plugin.Value)
{
Console.Write(" ");
if (attr.IsDefault)
Console.ForegroundColor = ConsoleColor.White;
Console.Write(attr.Name);
Console.ForegroundColor = color;
}
Console.WriteLine();
var desc = plugin.Key.GetAttributes<DescriptionAttribute>(null).FirstOrDefault();
if (desc != null)
Console.WriteLine(" [{0}]", desc.Description);
PrintParams(plugin.Key);
}
}
}