public void Test_0001_OnCrank_Percentage_100_0()
{
this.LogIn();
VariationSampler sampler = new VariationSamplerByKiiUser();
client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.RUNNING, 100, 0, null));
KiiExperiment experiment = KiiExperiment.GetByID("000001");
Variation variationA = experiment.Variations[0]; // 100%
Variation variationB = experiment.Variations[1]; // 0%
bool selectedA = false;
bool selectedB = false;
for (int i = 0; i < 1000; i++)
{
this.LogIn();
Variation variation = sampler.ChooseVariation(experiment, null);
if (variation == variationA)
{
selectedA = true;
}
else if (variation == variationB)
{
selectedB = true;
}
else
{
Assert.Fail("sampler returned unexpected variation");
}
}
Assert.IsTrue(selectedA);
Assert.IsFalse(selectedB);
}
public void Test_0000_OnCrank_Percentage_50_50()
{
this.LogIn();
VariationSampler sampler = new VariationSamplerByKiiUser();
client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.RUNNING, 50, 50, null));
KiiExperiment experiment = KiiExperiment.GetByID("000001");
Variation variationA = experiment.Variations[0]; // 50%
Variation variationB = experiment.Variations[1]; // 50%
int selectedCountA = 0;
int selectedCountB = 0;
for (int i = 0; i < 1000; i++)
{
this.LogIn();
Variation variation = sampler.ChooseVariation(experiment, null);
if (variation == variationA)
{
selectedCountA++;
}
else if (variation == variationB)
{
selectedCountB++;
}
else
{
Assert.Fail("sampler returned unexpected variation");
}
}
// This test will fail with a probability of 0.0017305361
Assert.IsTrue(450 < selectedCountA, "selectedCountA=" + selectedCountA);
Assert.IsTrue(550 > selectedCountA, "selectedCountA=" + selectedCountA);
Assert.IsTrue(450 < selectedCountB, "selectedCountB=" + selectedCountB);
Assert.IsTrue(550 > selectedCountB, "selectedCountB=" + selectedCountB);
}
public void Test_0004_OnCrank_Percentage_95_5()
{
System.Threading.Thread.Sleep(1);
VariationSampler sampler = new RandomVariationSampler();
client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.RUNNING, 95, 5, null));
KiiExperiment experiment = KiiExperiment.GetByID("000001");
Variation variationA = experiment.Variations[0]; // 95%
Variation variationB = experiment.Variations[1]; // 5%
int selectedCountA = 0;
int selectedCountB = 0;
for (int i = 0; i < 1000; i++)
{
System.Threading.Thread.Sleep(1);
Variation variation = sampler.ChooseVariation(experiment, null);
if (variation == variationA)
{
selectedCountA++;
}
else if (variation == variationB)
{
selectedCountB++;
}
else
{
Assert.Fail("sampler returned unexpected variation");
}
}
// This test will fail with a probability of 0.0349698573
Assert.IsTrue(35 < selectedCountB, "selectedCountB=" + selectedCountB);
Assert.IsTrue(65 > selectedCountB, "selectedCountB=" + selectedCountB);
}
/// <summary>
/// Get the variation applied to this trial.
/// Variation will be determined by specified rate of each variation in this experiment.
/// Current login user information will be used for sampling.
/// If the experiment has terminated with specified variant, the specified variant will be returned regardless of login user information.
/// If the experiment has finished without specified variant, fallback will be returned.
/// If the status of experiment is draft or paused, fallback will be returned.
/// </summary>
/// <remarks>
/// </remarks>
/// <returns>Applied variation for this trial.</returns>
/// <param name="fallback">The variation to return when failed to get the applied variation. If you want to detect error, you need to set null.</param>
public Variation GetAppliedVariation(Variation fallback)
{
if (KiiUser.CurrentUser == null)
{
return(fallback);
}
return(this.GetAppliedVariation(fallback, new VariationSamplerByKiiUser()));
}
public void Test_0012_GetByID_Status_Is_Out_Of_Range()
{
this.LogIn();
client.AddResponse(
200,
"{" +
" \"_id\" : \"ID-001\"," +
" \"description\" : \"Experiment for UT\"," +
" \"version\" : 2, " +
" \"status\" : 1000, " +
" \"conversionEvents\" : [ " +
" { \"name\" : \"Event-A\" }, " +
" { \"name\" : \"Event-B\" } " +
" ], " +
" \"variations\" : [ " +
" { " +
" \"name\" : \"A\", " +
" \"percentage\" : 50, " +
" \"variableSet\" : {\"message\" : \"set A\"} " +
" }, " +
" { " +
" \"name\" : \"B\", " +
" \"percentage\" : 50, " +
" \"variableSet\" : {\"message\" : \"set B\"} " +
" } " +
" ]" +
"}");
KiiExperiment experiment = KiiExperiment.GetByID(id);
// verify Experiment
Assert.AreEqual("ID-001", experiment.ID);
Assert.AreEqual("Experiment for UT", experiment.Description);
Assert.AreEqual(2, experiment.Version);
Assert.AreEqual(KiiExperimentStatus.PAUSED, experiment.Status);
// verify VariationA
Variation variationA = experiment.GetVariationByName("A");
Assert.AreEqual("A", variationA.Name);
Assert.AreEqual(50, variationA.Percentage);
Assert.AreEqual(
new JsonObject("{\"message\" : \"set A\"}").ToString(),
variationA.VariableSet.ToString());
ConversionEvent eventA = experiment.GetConversionEventByName("Event-A");
Assert.AreEqual("Event-A", eventA.Name);
// verify VariationB
Variation variationB = experiment.GetVariationByName("B");
Assert.AreEqual("B", variationB.Name);
Assert.AreEqual(50, variationB.Percentage);
Assert.AreEqual(
new JsonObject("{\"message\" : \"set B\"}").ToString(),
variationB.VariableSet.ToString());
ConversionEvent eventB = experiment.GetConversionEventByName("Event-B");
Assert.AreEqual("Event-B", eventB.Name);
}
public void Test_0007_GetAppliedVariation()
{
this.LogIn();
client.AddResponse(
200,
CreateKiiExperimentAsJsonString(id, description, version, KiiExperimentStatus.RUNNING,
variationNameA, eventNameA, 0, variableSetA,
variationNameB, eventNameB, 100, variableSetB,
variationNameA)
);
KiiExperiment experiment = KiiExperiment.GetByID(id);
Variation variation = experiment.GetAppliedVariation(null);
Assert.AreEqual(experiment.Variations[1], variation);
}
/// <summary>
/// Get the variation applied to this trial.
/// Sampler should return the variation according to the rate defined in this experiment.
/// If you use <see cref="VariationSamplerByKiiUser"/> with current login user, It will be same as <see cref="KiiExperiment.GetAppliedVariation(Variation)"/>
/// </summary>
/// <remarks>
/// </remarks>
/// <returns>Applied variation for this trial.</returns>
/// <param name="fallback">The variation to return when failed to get the applied variation. If you want to detect error, you need to set null.</param>
/// <param name="sampler">Variation sampler.</param>
public Variation GetAppliedVariation(Variation fallback, VariationSampler sampler)
{
if (sampler == null)
{
return(fallback);
}
try
{
return(sampler.ChooseVariation(this, fallback));
}
catch
{
return(fallback);
}
}
public void Test_0008_GetAppliedVariation_Fallback()
{
this.LogIn();
client.AddResponse(
200,
CreateKiiExperimentAsJsonString(id, description, version, KiiExperimentStatus.DRAFT,
variationNameA, eventNameA, 0, variableSetA,
variationNameB, eventNameB, 100, variableSetB,
variationNameA)
);
KiiExperiment experiment = KiiExperiment.GetByID(id);
Variation variation = experiment.GetAppliedVariation(null);
Assert.IsNull(variation);
}
public void Test_0008_OnCrank_With_Status_TERMINATED_Whit_ChosenVariation()
{
this.LogIn();
VariationSampler sampler = new VariationSamplerByKiiUser();
client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.TERMINATED, 50, 50, "B"));
KiiExperiment experiment = KiiExperiment.GetByID("000001");
Variation variationA = experiment.Variations[0]; // 50%
Variation variationB = experiment.Variations[1]; // 50%
Variation variation = sampler.ChooseVariation(experiment, null);
Assert.AreEqual(variationB, variation);
variation = sampler.ChooseVariation(experiment, variationA);
Assert.AreEqual(variationB, variation);
variation = sampler.ChooseVariation(experiment, variationB);
Assert.AreEqual(variationB, variation);
}
public void Test_0009_GetAppliedVariation_With_CustomSampler()
{
this.LogIn();
client.AddResponse(
200,
CreateKiiExperimentAsJsonString(id, description, version, KiiExperimentStatus.RUNNING,
variationNameA, eventNameA, 0, variableSetA,
variationNameB, eventNameB, 100, variableSetB,
variationNameA)
);
KiiExperiment experiment = KiiExperiment.GetByID(id);
Variation variation = experiment.GetAppliedVariation(null, new CustomVariationSampler());
// get VariationsA even its percentage is 0.
Assert.AreEqual(experiment.Variations[0], variation);
}
/// <summary>
/// Do sampling. Returns random variation based on the percentage configured in developer portal.
/// If sampling is failed, returns the fallback.
/// If the experiment has terminated and fixed variation has chosen, returns chosen variation.
/// Returned variation is same as <see cref="KiiExperiment.ChosenVariation"/>
/// </summary>
/// <remarks>
/// </remarks>
/// <returns>Applied variation for this time.</returns>
/// <param name="experiment">that requires sampling.</param>
/// <param name="fallback">The variation to return when failed to get the applied variation.</param>
public Variation ChooseVariation(KiiExperiment experiment, Variation fallback)
{
KiiUser user = KiiUser.CurrentUser;
if (user == null)
{
throw new InvalidOperationException(ErrorInfo.UTILS_NO_LOGIN);
}
switch (experiment.Status)
{
case KiiExperimentStatus.RUNNING:
int totalPercentage = 0;
foreach (Variation v in experiment.Variations)
{
totalPercentage += v.Percentage;
}
int random = new Random(user.Uri.GetHashCode()).Next(totalPercentage);
int i = 0;
int accumulated = 0;
foreach (Variation v in experiment.Variations)
{
if (random < accumulated + v.Percentage)
{
return(v);
}
accumulated += v.Percentage;
i++;
}
throw new InvalidOperationException("Unexpected error.");
case KiiExperimentStatus.DRAFT:
return(fallback);
case KiiExperimentStatus.PAUSED:
return(fallback);
case KiiExperimentStatus.TERMINATED:
if (experiment.ChosenVariation != null)
{
return(experiment.ChosenVariation);
}
return(fallback);
default:
throw new InvalidOperationException("Unknown status!");
}
}
public void Test_0007_OnCrank_With_Status_TERMINATED_Whitout_ChosenVariation()
{
System.Threading.Thread.Sleep(1);
VariationSampler sampler = new RandomVariationSampler();
client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.TERMINATED, 50, 50, null));
KiiExperiment experiment = KiiExperiment.GetByID("000001");
Variation variationA = experiment.Variations[0]; // 50%
Variation variationB = experiment.Variations[1]; // 50%
Variation variation = sampler.ChooseVariation(experiment, null);
Assert.IsNull(variation);
variation = sampler.ChooseVariation(experiment, variationA);
Assert.AreEqual(variationA, variation);
variation = sampler.ChooseVariation(experiment, variationB);
Assert.AreEqual(variationB, variation);
}
public void Test_0005_GetVariationByName()
{
this.LogIn();
client.AddResponse(
200,
CreateKiiExperimentAsJsonString(id, description, version, status,
variationNameA, eventNameA, percentageA, variableSetA,
variationNameB, eventNameB, percentageB, variableSetB,
variationNameA)
);
KiiExperiment experiment = KiiExperiment.GetByID(id);
Variation variation = experiment.GetVariationByName(variationNameB);
Assert.AreEqual(experiment.Variations[1], variation);
variation = experiment.GetVariationByName("hoge");
Assert.IsNull(variation);
}
/// <summary>
/// Do sampling. Returns random variation based on the percentage configured in developer portal.
/// If sampling is failed, returns the fallback.
/// If the experiment has terminated and fixed variation has chosen, returns chosen variation.
/// Returned variation is same as <see cref="KiiExperiment.ChosenVariation"/>
/// </summary>
/// <remarks>
/// </remarks>
/// <returns>Applied variation for this time.</returns>
/// <param name="experiment">that requires sampling.</param>
/// <param name="fallback">The variation to return when failed to get the applied variation.</param>
public Variation ChooseVariation(KiiExperiment experiment, Variation fallback)
{
switch (experiment.Status)
{
case KiiExperimentStatus.RUNNING:
int totalPercentage = 0;
foreach (Variation v in experiment.Variations)
{
totalPercentage += v.Percentage;
}
int random = 0;
lock (this)
{
random = this.mRandomGenerator.Next(totalPercentage);
}
int accumulated = 0;
foreach (Variation v in experiment.Variations)
{
accumulated += v.Percentage;
if (random < accumulated)
{
return(v);
}
}
throw new InvalidOperationException("Unexpected error.");
case KiiExperimentStatus.DRAFT:
return(fallback);
case KiiExperimentStatus.PAUSED:
return(fallback);
case KiiExperimentStatus.TERMINATED:
if (experiment.ChosenVariation != null)
{
return(experiment.ChosenVariation);
}
return(fallback);
default:
throw new InvalidOperationException("Unknown status!");
}
}
public void Test_0005_EventForConversionWhenStatusIsTerminated()
{
this.LogIn();
int experimentVersion = 10;
client.AddResponse(200, CreateKiiExperimentAsJsonString(experimentVersion, KiiExperimentStatus.TERMINATED, "UI-A", "EV-A", "UI-B", "EV-B"));
KiiExperiment experiment = KiiExperiment.GetByID("000001");
Variation variationA = experiment.Variations[0];
Variation variationB = experiment.Variations[1];
ConversionEvent eventA = experiment.ConversionEvents[0];
ConversionEvent eventB = experiment.ConversionEvents[1];
KiiEvent kiiEventA = variationA.EventForConversion(eventA);
Assert.IsInstanceOfType(typeof(KiiEvent.NullKiiEvent), kiiEventA);
KiiEvent kiiEventB = variationB.EventForConversion(eventB);
Assert.IsInstanceOfType(typeof(KiiEvent.NullKiiEvent), kiiEventB);
}
public void Test_0001_EventForConversion()
{
this.LogIn();
int experimentVersion = 10;
client.AddResponse(200, CreateKiiExperimentAsJsonString(experimentVersion, KiiExperimentStatus.RUNNING, "UI-A", "EV-A", "UI-B", "EV-B"));
KiiExperiment experiment = KiiExperiment.GetByID("000001");
Variation variationA = experiment.Variations[0];
Variation variationB = experiment.Variations[1];
ConversionEvent eventA = experiment.ConversionEvents[0];
ConversionEvent eventB = experiment.ConversionEvents[1];
KiiEvent kiiEventA = variationA.EventForConversion(0);
Assert.AreEqual(variationA.Name, kiiEventA["variationName"]);
Assert.AreEqual(eventA.Name, kiiEventA["conversionEvent"]);
Assert.AreEqual(experimentVersion, kiiEventA["version"]);
KiiEvent kiiEventB = variationB.EventForConversion(1);
Assert.AreEqual(variationB.Name, kiiEventB["variationName"]);
Assert.AreEqual(eventB.Name, kiiEventB["conversionEvent"]);
Assert.AreEqual(experimentVersion, kiiEventB["version"]);
}
public Variation ChooseVariation(KiiExperiment experiment, Variation fallback)
{
return(experiment.Variations[0]);
}