public virtual void Publish(IExperimentResult <TPublish> results)
{
if (Publisher != null)
{
Publisher.Publish(results);
}
}
public void Publish <T>(IExperimentResult <T> results)
{
// How can we Unit Test this? Could do integration test but would be overkill
using (var udp = new StatsdUDP(hostName, port))
{
var statsd = new Statsd(udp, prefix);
AddObservationStats(results.Name, results.Control, statsd);
foreach (var kvp in results.Candidates)
{
AddObservationStats(results.Name, kvp.Value, statsd);
}
statsd.Send();
}
}
public virtual void Publish <T>(IExperimentResult <T> results)
{
Console.WriteLine("*************** Experiment '{0}' Results ***************", results.Name);
PublishObservation(results.Control);
foreach (var obs in results.Candidates)
{
PublishObservation(obs.Value);
}
Console.WriteLine(messages.ToString());
var stars = new StringBuilder();
for (int i = 0; i < results.Name.Length; i++)
{
stars.Append('*');
}
Console.WriteLine("*****************************************************" + stars.ToString());
messages.Clear();
}
private void TryPublish(IExperimentResult <TPublish> results)
{
CurrentState.CurrentStep = Operations.Publish;
TryOp(() => { Publish(results); return(0); });
}
private T RunControl(IExperimentResult <TPublish> results, out Exception controlException)
{
controlException = null;
CurrentState.Name = ControlName;
CurrentState.CurrentStep = Operations.Control;
if (Steps.Control == null)
{
// Can there be valid experiments with no Control?? Dunno but seems restrictive to enfore it
// if so, return default(T) instead of Exception
return(default(T));
// Throw exception if we want to enforce having a Control
//throw new InvalidOperationException(
// "The Control was never set for this Experiment! Can't run an Experiment without a Control!");
}
IExperimentError stepError = null;
object context = null;
T result = default(T);
TPublish value = default(TPublish);
try
{
context = TrySetContext();
CurrentState.Context = context;
}
catch (StepFailedException sfe)
{
stepError = sfe.ExperimentError;
TryOnError(stepError);
}
try
{
TrySetup();
}
catch (StepFailedException sfe)
{
stepError = sfe.ExperimentError;
TryOnError(stepError);
}
var timer = new Stopwatch();
CurrentState.CurrentStep = Operations.Control;
try
{
timer.Start();
result = Steps.Control();
timer.Stop();
}
catch (Exception e)
{
timer.Stop();
controlException = e; //need to throw this at the end
var error = new ExperimentError
{
LastException = e,
LastStep = Operations.Control,
ErrorMessage = "An Exception was thrown running the Control! Exception message: "
+ e.Message,
ExperimentName = ControlName
};
TryOnError(error);//TODO: Should we run OnError on Control??
results.Control = new Observation <TPublish>
{
ExperimentError = error,
ElapsedMilliseconds = timer.ElapsedMilliseconds,
Context = context,
Name = ControlName,
IsMismatched = false,
ExceptionThrown = true
};
return(result);
}
try
{
value = TryPrepare(result);
}
catch (StepFailedException sfe)
{
stepError = sfe.ExperimentError;
TryOnError(stepError);
//We must continue on because we've already gotten a result
}
try
{
TryTeardown();
}
catch (StepFailedException sfe)
{
stepError = sfe.ExperimentError;
TryOnError(stepError);
}
results.Control = new Observation <TPublish>
{
ElapsedMilliseconds = timer.ElapsedMilliseconds,
Context = context,
Name = ControlName,
IsMismatched = false, //Control can't be mismatched
ExceptionThrown = stepError != null,
ExperimentError = stepError,
Value = value
};
return(result);
}
private bool RunCandidates(IExperimentResult <TPublish> results, T controlResult, Exception controlException)
{
bool ran = false;
var candidates = Steps.GetCandidates();
if (candidates != null)
{
var timer = new Stopwatch();
foreach (var candidate in candidates)
{
T candResult = default(T);
TPublish candValue = default(TPublish);
bool mismatched = false;
try
{
//Should this be a State machine?
CurrentState.Name = candidate.Key;
if (TryPreCondition() && candidate.Value != null)
{
ran = true;
var context = TrySetContext();
CurrentState.Context = context;
TrySetup();
timer.Restart();
candResult = TryCandidate(candidate.Value);
timer.Stop();
if (!TryIgnore(controlResult, candResult) &&
(controlException != null || !TryAreEqual(controlResult, candResult)))
{
mismatched = true;
TryOnMismatch(controlResult, candResult, controlException, null);
}
candValue = TryPrepare(candResult);
TryTeardown();
if (!results.Candidates.ContainsKey(candidate.Key))
{
results.Candidates.Add(candidate.Key,
new Observation <TPublish>
{
Value = candValue,
ElapsedMilliseconds = timer.ElapsedMilliseconds,
ElapsedTime = timer.Elapsed,
Context = context,
IsMismatched = mismatched,
ExceptionThrown = false,
Name = candidate.Key
});
}
else
{
//Duplicate keys... should not be possible
}
}
}
catch (StepFailedException sfe)
{
timer.Stop();
//it's possible for this to throw, which would result in an internal exception
TryOnError(sfe.ExperimentError);
if (!exceptionComparer.Equals(sfe.ExperimentError.LastException, controlException))
{
//It's a mismatch if the exceptions don't match
mismatched = true;
TryOnMismatch(controlResult, candResult,
controlException, sfe.ExperimentError.LastException); //potential throw
}
var errResult = new Observation <TPublish>
{
ExperimentError = sfe.ExperimentError,
Value = candValue,
ElapsedMilliseconds = timer.ElapsedMilliseconds,
ExceptionThrown = true,
Name = candidate.Key,
IsMismatched = mismatched
};
if (results.Candidates.ContainsKey(results.LastState.Name))
{
results.Candidates[results.LastState.Name] = errResult;
}
else
{
results.Candidates.Add(results.LastState.Name, errResult);
}
}
}
}
return(ran);
}
