public static void Register(string name, string description, Func<string> identify = null, Func<Experiment, bool> conclude = null, Func<Experiment, IOrderedEnumerable<KeyValuePair<int, double>>> score = null, Func<string, int, int> splitOn = null, object[] alternatives = null, params string[] metrics)
{
var experiment = new Experiment(name, description, identify, conclude, score, splitOn, alternatives, metrics);
ExperimentRepository.Save(experiment);
foreach (var metric in experiment.Metrics.Select(GetOrRegisterMetric))
{
metric.Hook += (sender, args) => Track(experiment, args);
}
}
public virtual Score Score(Experiment experiment, double probability = 90.0)
{
var score = new Score();
// sort by conversion rate
var sorted = experiment.Score(experiment);
var @base = sorted.Skip(1).Take(1).First(); // sorted[-2]
// calculate z-score
var cohort = experiment.ParticipantsByAlternative();
var pc = @base.Value;
var nc = cohort[@base.Key];
foreach (var alt in sorted)
{
var p = alt.Value;
var index = alt.Key;
var n = cohort[index];
var zscore = (p - pc)/((p * (1 - p) / n) + (pc * (1 - pc) / nc)).Abs().Pow(0.5);
zscore = double.IsNaN(zscore) ? 0 : zscore;
score.Alternatives.Add(new Alternative
{
Index = index,
Name = experiment.Alternatives[index - 1].ToString(),
Label = "Option " + (char)(index + 64),
Value = alt.Value,
ZScore = zscore,
Probability = ProbabilityOfScore(zscore)
});
}
score.Base = score.Alternatives[@base.Key - 1];
// difference is measured from least performant
KeyValuePair<int, double> least;
if ((least = sorted.FirstOrDefault(kv => kv.Value > 0)).Value > 0)
{
foreach (var alt in sorted.Where(alt => alt.Value > least.Value))
{
score.Alternatives[alt.Key].Difference = (alt.Value - least.Value) / least.Value * 100;
}
score.Least = score.Alternatives[least.Key - 1];
}
// best alternative is one with highest conversion rate (best shot).
if (sorted.Last().Value > 0.0)
{
score.Best = score.Alternatives[sorted.Last().Key - 1];
}
// choice alternative can only pick best if we have high probability (>90%)
score.Choice = experiment.Outcome.HasValue
? score.Alternatives[experiment.Outcome.Value - 1]
: (score.Best != null && score.Best.Probability >= probability ? score.Best : null);
return score;
}
public void Save(Experiment experiment)
{
var exists = GetByName(experiment.Name);
var db = _connector();
if(exists == null)
{
db.Insert(experiment);
}
else
{
db.Update(experiment);
}
}
public static IEnumerable<AlternativeViewModel> ProjectAlternatives(Experiment experiment, Score score)
{
var index = 1;
foreach (var alternative in experiment.Alternatives)
{
var value = alternative.ToString();
var vm = new AlternativeViewModel
{
Name = "Option " + (char)(index + 64),
Participants = experiment.ParticipantsByAlternative()[index],
Converted = experiment.ConvertedByAlternative()[index],
ConversionRate = experiment.ConversionRateByAlternative()[index],
Showing = experiment.AlternativeValue.ToString() == value,
Choice = score.Choice != null && score.Choice.Index == index,
Value = value
};
index++;
yield return vm;
}
}
private static void Track(Experiment experiment, MetricEventArgs args)
{
if (!experiment.IsActive)
{
return;
}
args = args;
experiment.CurrentParticipant.Conversions++;
experiment.CurrentParticipant.Seen++;
var index = experiment.AlternativeIndex;
//def track!(metric_id, timestamp, count, *args)
// return unless active?
// identity = identity() rescue nil
// if identity
// return if connection.ab_showing(@id, identity)
// index = alternative_for(identity)
// connection.ab_add_conversion @id, index, identity, count
// check_completion!
// end
// end
}
public void Save(Experiment experiment)
{
var key = experiment.Name;
Inner.AddOrUpdate(key, experiment, (n, m) => m);
}
public void Save(Experiment experiment)
{
var key = experiment.Name;
Inner.AddOrUpdate(key, experiment, (n, m) => m);
}