public virtual Score Score(Experiment experiment, double probability = 90.0)
{
var score = new Score();
// sort by conversion rate
var sorted = experiment.Score(experiment);
var @base = sorted.Skip(1).Take(1).First(); // sorted[-2]
// calculate z-score
var cohort = experiment.ParticipantsByAlternative();
var pc = @base.Value;
var nc = cohort[@base.Key];
foreach (var alt in sorted)
{
var p = alt.Value;
var index = alt.Key;
var n = cohort[index];
var zscore = (p - pc) / ((p * (1 - p) / n) + (pc * (1 - pc) / nc)).Abs().Pow(0.5);
zscore = double.IsNaN(zscore) ? 0 : zscore;
score.Alternatives.Add(new Alternative
{
Index = index,
Name = experiment.Alternatives[index - 1].ToString(),
Label = "Option " + (char)(index + 64),
Value = alt.Value,
ZScore = zscore,
Probability = ProbabilityOfScore(zscore)
});
}
score.Base = score.Alternatives[@base.Key - 1];
// difference is measured from least performant
KeyValuePair <int, double> least;
if ((least = sorted.FirstOrDefault(kv => kv.Value > 0)).Value > 0)
{
foreach (var alt in sorted.Where(alt => alt.Value > least.Value))
{
score.Alternatives[alt.Key].Difference = (alt.Value - least.Value) / least.Value * 100;
}
score.Least = score.Alternatives[least.Key - 1];
}
// best alternative is one with highest conversion rate (best shot).
if (sorted.Last().Value > 0.0)
{
score.Best = score.Alternatives[sorted.Last().Key - 1];
}
// choice alternative can only pick best if we have high probability (>90%)
score.Choice = experiment.Outcome.HasValue
? score.Alternatives[experiment.Outcome.Value - 1]
: (score.Best != null && score.Best.Probability >= probability ? score.Best : null);
return(score);
}
public virtual Score Score(Experiment experiment, double probability = 90.0)
{
var score = new Score();
// sort by conversion rate
var sorted = experiment.Score(experiment);
var @base = sorted.Skip(1).Take(1).First(); // sorted[-2]
// calculate z-score
var cohort = experiment.ParticipantsByAlternative();
var pc = @base.Value;
var nc = cohort[@base.Key];
foreach (var alt in sorted)
{
var p = alt.Value;
var index = alt.Key;
var n = cohort[index];
var zscore = (p - pc)/((p * (1 - p) / n) + (pc * (1 - pc) / nc)).Abs().Pow(0.5);
zscore = double.IsNaN(zscore) ? 0 : zscore;
score.Alternatives.Add(new Alternative
{
Index = index,
Name = experiment.Alternatives[index - 1].ToString(),
Label = "Option " + (char)(index + 64),
Value = alt.Value,
ZScore = zscore,
Probability = ProbabilityOfScore(zscore)
});
}
score.Base = score.Alternatives[@base.Key - 1];
// difference is measured from least performant
KeyValuePair<int, double> least;
if ((least = sorted.FirstOrDefault(kv => kv.Value > 0)).Value > 0)
{
foreach (var alt in sorted.Where(alt => alt.Value > least.Value))
{
score.Alternatives[alt.Key].Difference = (alt.Value - least.Value) / least.Value * 100;
}
score.Least = score.Alternatives[least.Key - 1];
}
// best alternative is one with highest conversion rate (best shot).
if (sorted.Last().Value > 0.0)
{
score.Best = score.Alternatives[sorted.Last().Key - 1];
}
// choice alternative can only pick best if we have high probability (>90%)
score.Choice = experiment.Outcome.HasValue
? score.Alternatives[experiment.Outcome.Value - 1]
: (score.Best != null && score.Best.Probability >= probability ? score.Best : null);
return score;
}
