public virtual Score Score(Experiment experiment, double probability = 90.0) { var score = new Score(); // sort by conversion rate var sorted = experiment.Score(experiment); var @base = sorted.Skip(1).Take(1).First(); // sorted[-2] // calculate z-score var cohort = experiment.ParticipantsByAlternative(); var pc = @base.Value; var nc = cohort[@base.Key]; foreach (var alt in sorted) { var p = alt.Value; var index = alt.Key; var n = cohort[index]; var zscore = (p - pc) / ((p * (1 - p) / n) + (pc * (1 - pc) / nc)).Abs().Pow(0.5); zscore = double.IsNaN(zscore) ? 0 : zscore; score.Alternatives.Add(new Alternative { Index = index, Name = experiment.Alternatives[index - 1].ToString(), Label = "Option " + (char)(index + 64), Value = alt.Value, ZScore = zscore, Probability = ProbabilityOfScore(zscore) }); } score.Base = score.Alternatives[@base.Key - 1]; // difference is measured from least performant KeyValuePair <int, double> least; if ((least = sorted.FirstOrDefault(kv => kv.Value > 0)).Value > 0) { foreach (var alt in sorted.Where(alt => alt.Value > least.Value)) { score.Alternatives[alt.Key].Difference = (alt.Value - least.Value) / least.Value * 100; } score.Least = score.Alternatives[least.Key - 1]; } // best alternative is one with highest conversion rate (best shot). if (sorted.Last().Value > 0.0) { score.Best = score.Alternatives[sorted.Last().Key - 1]; } // choice alternative can only pick best if we have high probability (>90%) score.Choice = experiment.Outcome.HasValue ? score.Alternatives[experiment.Outcome.Value - 1] : (score.Best != null && score.Best.Probability >= probability ? score.Best : null); return(score); }
public virtual Score Score(Experiment experiment, double probability = 90.0) { var score = new Score(); // sort by conversion rate var sorted = experiment.Score(experiment); var @base = sorted.Skip(1).Take(1).First(); // sorted[-2] // calculate z-score var cohort = experiment.ParticipantsByAlternative(); var pc = @base.Value; var nc = cohort[@base.Key]; foreach (var alt in sorted) { var p = alt.Value; var index = alt.Key; var n = cohort[index]; var zscore = (p - pc)/((p * (1 - p) / n) + (pc * (1 - pc) / nc)).Abs().Pow(0.5); zscore = double.IsNaN(zscore) ? 0 : zscore; score.Alternatives.Add(new Alternative { Index = index, Name = experiment.Alternatives[index - 1].ToString(), Label = "Option " + (char)(index + 64), Value = alt.Value, ZScore = zscore, Probability = ProbabilityOfScore(zscore) }); } score.Base = score.Alternatives[@base.Key - 1]; // difference is measured from least performant KeyValuePair<int, double> least; if ((least = sorted.FirstOrDefault(kv => kv.Value > 0)).Value > 0) { foreach (var alt in sorted.Where(alt => alt.Value > least.Value)) { score.Alternatives[alt.Key].Difference = (alt.Value - least.Value) / least.Value * 100; } score.Least = score.Alternatives[least.Key - 1]; } // best alternative is one with highest conversion rate (best shot). if (sorted.Last().Value > 0.0) { score.Best = score.Alternatives[sorted.Last().Key - 1]; } // choice alternative can only pick best if we have high probability (>90%) score.Choice = experiment.Outcome.HasValue ? score.Alternatives[experiment.Outcome.Value - 1] : (score.Best != null && score.Best.Probability >= probability ? score.Best : null); return score; }
public static IDictionary <int, double> ConversionRateByAlternative(this Experiment experiment) { var participants = experiment.ParticipantsByAlternative(); var converted = experiment.ConvertedByAlternative(); var hash = experiment.EmptyHash <double>(); for (var i = 1; i <= experiment.Alternatives.Count(); i++) { var conversionRate = participants[i] > 0 ? (converted[i] / (double)participants[i]) * 100 : 0; hash[i] = conversionRate; } return(hash); }
public static IEnumerable<AlternativeViewModel> ProjectAlternatives(Experiment experiment, Score score) { var index = 1; foreach (var alternative in experiment.Alternatives) { var value = alternative.ToString(); var vm = new AlternativeViewModel { Name = "Option " + (char)(index + 64), Participants = experiment.ParticipantsByAlternative()[index], Converted = experiment.ConvertedByAlternative()[index], ConversionRate = experiment.ConversionRateByAlternative()[index], Showing = experiment.AlternativeValue.ToString() == value, Choice = score.Choice != null && score.Choice.Index == index, Value = value }; index++; yield return vm; } }
public static IEnumerable <AlternativeViewModel> ProjectAlternatives(Experiment experiment, Score score) { var index = 1; foreach (var alternative in experiment.Alternatives) { var value = alternative.ToString(); var vm = new AlternativeViewModel { Name = "Option " + (char)(index + 64), Participants = experiment.ParticipantsByAlternative()[index], Converted = experiment.ConvertedByAlternative()[index], ConversionRate = experiment.ConversionRateByAlternative()[index], Showing = experiment.AlternativeValue.ToString() == value, Choice = score.Choice != null && score.Choice.Index == index, Value = value }; index++; yield return(vm); } }