/// <summary>
/// Mark this user as having converted for the specified tests.
/// </summary>
public void ScoreConversion(string testName)
{
ABUser user = IdentifyUser();
if (!user.Tests.Contains(testName) || user.Conversions.Contains(testName))
{
// not part of the test or already scored.
return;
}
SerializableDictionary <string, Experiment> tests = GetTests();
if (tests.ContainsKey(testName))
{
Experiment t = tests[testName];
ABAlternative choice = t.GetUserAlternative(user.ID);
choice.ScoreConversion();
user.Conversions.Add(testName);
user.SaveToCookie();
SaveTests(tests);
}
}
/// <summary>
/// Given a userID, return the appropriate alternative.
/// </summary>
/// <param name="userID"></param>
/// <returns></returns>
public ABAlternative GetUserAlternative(int userID)
{
int index = userID % Alternatives.Count;
ABAlternative choice = Alternatives[index];
choice.Index = index;
return(choice);
}
public string GetResultDescription(Experiment test)
{
double p;
bool testAssumptionsUpheld = false;
try
{
p = GetPValue(test, out testAssumptionsUpheld);
}
catch (Exception e)
{
return(e.Message);
}
StringBuilder builder = new StringBuilder();
if (!testAssumptionsUpheld)
{
builder.Append("Caution: the sample did not conform to the expected cell frequency condition! ");
}
ABAlternative best = test.GetBestAlternative();
ABAlternative worst = test.GetWorstAlternative();
builder.Append(String.Format(@"
The best alternative you have is: [{0}], which had
{1} conversions from {2} participants
({3}). "
, best.Content
, best.Conversions
, best.Participants
, best.PrettyConversionRate
));
if (p == 1)
{
builder.Append("However, this result is not statistically significant.");
}
else
{
builder.Append(String.Format(@"
This difference is <b>{0} likely to be statistically significant (p <= {2})</b>, which means you can be
{1} that it is the result of your alternatives actually mattering, rather than
being due to random chance. However, this statistical test can't measure how likely the currently
observed magnitude of the difference is to be accurate or not. It only says ""better"", not ""better
by so much"". " ,
//Percentages[p],
ToPercentageString(p),
Descriptions[p],
p
));
}
return(builder.ToString());
}
public ABAlternative GetWorstAlternative()
{
ABAlternative best = null;
foreach (ABAlternative alt in Alternatives)
{
if (best == null || alt.ConversionRate <= best.ConversionRate)
{
best = alt;
}
}
return(best);
}
private void ScoreParticipation(string testName, ABAlternative choice)
{
SerializableDictionary <string, Experiment> tests = GetTests();
if (tests.ContainsKey(testName))
{
Experiment t = tests[testName];
foreach (ABAlternative a in t.Alternatives)
{
if (a.Content == choice.Content)
{
a.Participants += 1;
break;
}
}
SaveTests(tests);
}
}
/// <summary>
/// For the specified test, pick an alternative to always show this user, and return that alternative.
/// </summary>
/// <param name="test"></param>
/// <returns></returns>
public ABAlternative GetUserAlternative(Experiment test)
{
//complete an experiment as soon as we reach our required sample size...
if (test.IsComplete)
{
return(test.GetBestAlternative());
}
ABUser user = IdentifyUser();
ABAlternative choice = test.GetUserAlternative(user.ID);
if (!user.Tests.Contains(test.TestName) && !IsBotRequest()) //don't score the participation more than once for an identified user (don't score for bots either)
{
choice.ScoreParticipation();
user.Tests.Add(test.TestName);
user.SaveToCookie();
//persist the new participant count to the file store...
ScoreParticipation(test.TestName, choice);
}
return(choice);
}
private void ScoreParticipation(string testName, ABAlternative choice)
{
SerializableDictionary<string, Experiment> tests = GetTests();
if (tests.ContainsKey(testName))
{
Experiment t = tests[testName];
foreach (ABAlternative a in t.Alternatives)
{
if (a.Content == choice.Content)
{
a.Participants += 1;
break;
}
}
SaveTests(tests);
}
}
public string GetResultDescription(Experiment test)
{
double p;
try
{
p = GetPValue(test);
}
catch (Exception e)
{
return(e.Message);
}
StringBuilder builder = new StringBuilder();
//CF: This is wrong. The assumption for a two-proportion z-test is that each sample have at least 10 succeses AND 10 failures!
//if (Alternatives[0].Participants < 10 || Alternatives[1].Participants < 10)
if (test.Alternatives.Count(x => !x.SampleMeetsTestAssumtions) > 0)
{
builder.Append("Take these results with a grain of salt since your samples do not meet the required assumptions: ");
}
ABAlternative best = test.GetBestAlternative();
ABAlternative worst = test.GetWorstAlternative();
builder.Append(String.Format(@"
The best alternative you have is: [{0}], which had
{1} conversions from {2} participants
({3}). The other alternative was [{4}],
which had {5} conversions from {6} participants
({7}). "
, best.Content
, best.Conversions
, best.Participants
, best.PrettyConversionRate
, worst.Content
, worst.Conversions
, worst.Participants
, worst.PrettyConversionRate
));
if (p == 1)
{
builder.Append("However, this difference is not statistically significant.");
}
else
{
builder.Append(String.Format(@"
This difference is <b>{0} likely to be statistically significant (p <= {2})</b>, which means you can be
{1} that it is the result of your alternatives actually mattering, rather than
being due to random chance. However, this statistical test can't measure how likely the currently
observed magnitude of the difference is to be accurate or not. It only says ""better"", not ""better
by so much"". " ,
//Percentages[p],
ToPercentageString(p),
Descriptions[p],
p
));
}
return(builder.ToString());
}
