public void Test_0000_OnCrank_Percentage_50_50()
{
this.LogIn();
VariationSampler sampler = new VariationSamplerByKiiUser();
client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.RUNNING, 50, 50, null));
KiiExperiment experiment = KiiExperiment.GetByID("000001");
Variation variationA = experiment.Variations[0]; // 50%
Variation variationB = experiment.Variations[1]; // 50%
int selectedCountA = 0;
int selectedCountB = 0;
for (int i = 0; i < 1000; i++)
{
this.LogIn();
Variation variation = sampler.ChooseVariation(experiment, null);
if (variation == variationA)
{
selectedCountA++;
}
else if (variation == variationB)
{
selectedCountB++;
}
else
{
Assert.Fail("sampler returned unexpected variation");
}
}
// This test will fail with a probability of 0.0017305361
Assert.IsTrue(450 < selectedCountA, "selectedCountA=" + selectedCountA);
Assert.IsTrue(550 > selectedCountA, "selectedCountA=" + selectedCountA);
Assert.IsTrue(450 < selectedCountB, "selectedCountB=" + selectedCountB);
Assert.IsTrue(550 > selectedCountB, "selectedCountB=" + selectedCountB);
}
public void Test_0004_OnCrank_Percentage_95_5()
{
System.Threading.Thread.Sleep(1);
VariationSampler sampler = new RandomVariationSampler();
client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.RUNNING, 95, 5, null));
KiiExperiment experiment = KiiExperiment.GetByID("000001");
Variation variationA = experiment.Variations[0]; // 95%
Variation variationB = experiment.Variations[1]; // 5%
int selectedCountA = 0;
int selectedCountB = 0;
for (int i = 0; i < 1000; i++)
{
System.Threading.Thread.Sleep(1);
Variation variation = sampler.ChooseVariation(experiment, null);
if (variation == variationA)
{
selectedCountA++;
}
else if (variation == variationB)
{
selectedCountB++;
}
else
{
Assert.Fail("sampler returned unexpected variation");
}
}
// This test will fail with a probability of 0.0349698573
Assert.IsTrue(35 < selectedCountB, "selectedCountB=" + selectedCountB);
Assert.IsTrue(65 > selectedCountB, "selectedCountB=" + selectedCountB);
}
/// <summary>
/// Get the experiment having the specified id in background.
/// </summary>
/// <remarks>
/// NOTE: This api access to server. Should not be executed in UI/Main thread.
/// </remarks>
/// <returns>Experiment which has the specified id.</returns>
/// <param name="experimentID">Experiment id.</param>
/// <param name="callback">Called on completion of get experiment.</param>
public static void GetByID(String experimentID, KiiExperimentCallback callback)
{
if (!IsValidExperimentID(experimentID))
{
callback(null, new ArgumentException("Experiment id is invalid"));
return;
}
Utils.CheckInitialize(false);
string getUrl = Utils.Path(Kii.BaseUrl, "apps", Kii.AppId, "buckets", "_experiments", "objects", experimentID);
KiiHttpClient client = Kii.AsyncHttpClientFactory.Create(getUrl, Kii.AppId, Kii.AppKey, KiiHttpMethod.GET);
KiiCloudEngine.SetAuthBearer(client);
client.SendRequest((ApiResponse response, Exception e) =>
{
if (e != null)
{
callback(null, e);
return;
}
else
{
try
{
KiiExperiment kiiExperiment = CreateFromResponse(response);
callback(kiiExperiment, null);
}
catch (Exception ex)
{
callback(null, ex);
}
}
});
}
public void Test_0001_OnCrank_Percentage_100_0()
{
this.LogIn();
VariationSampler sampler = new VariationSamplerByKiiUser();
client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.RUNNING, 100, 0, null));
KiiExperiment experiment = KiiExperiment.GetByID("000001");
Variation variationA = experiment.Variations[0]; // 100%
Variation variationB = experiment.Variations[1]; // 0%
bool selectedA = false;
bool selectedB = false;
for (int i = 0; i < 1000; i++)
{
this.LogIn();
Variation variation = sampler.ChooseVariation(experiment, null);
if (variation == variationA)
{
selectedA = true;
}
else if (variation == variationB)
{
selectedB = true;
}
else
{
Assert.Fail("sampler returned unexpected variation");
}
}
Assert.IsTrue(selectedA);
Assert.IsFalse(selectedB);
}
public void Test_0001_GetByID()
{
this.LogIn();
client.AddResponse(
200,
CreateKiiExperimentAsJsonString(id, description, version, status,
variationNameA, eventNameA, percentageA, variableSetA,
variationNameB, eventNameB, percentageB, variableSetB,
variationNameA)
);
KiiExperiment experiment = KiiExperiment.GetByID(id);
// verify Experiment
Assert.AreEqual(id, experiment.ID);
Assert.AreEqual(description, experiment.Description);
Assert.AreEqual(version, experiment.Version);
Assert.AreEqual(status, experiment.Status);
Assert.AreEqual(experiment.Variations[0], experiment.ChosenVariation);
// verify VariationA
Assert.AreEqual(variationNameA, experiment.Variations[0].Name);
Assert.AreEqual(percentageA, experiment.Variations[0].Percentage);
Assert.AreEqual(new JsonObject(variableSetA).ToString(), experiment.Variations[0].VariableSet.ToString());
Assert.AreEqual(eventNameA, experiment.ConversionEvents[0].Name);
// verify VariationB
Assert.AreEqual(variationNameB, experiment.Variations[1].Name);
Assert.AreEqual(percentageB, experiment.Variations[1].Percentage);
Assert.AreEqual(new JsonObject(variableSetB).ToString(), experiment.Variations[1].VariableSet.ToString());
Assert.AreEqual(eventNameB, experiment.ConversionEvents[1].Name);
}
public void Test_0012_GetByID_Status_Is_Out_Of_Range()
{
this.LogIn();
client.AddResponse(
200,
"{" +
" \"_id\" : \"ID-001\"," +
" \"description\" : \"Experiment for UT\"," +
" \"version\" : 2, " +
" \"status\" : 1000, " +
" \"conversionEvents\" : [ " +
" { \"name\" : \"Event-A\" }, " +
" { \"name\" : \"Event-B\" } " +
" ], " +
" \"variations\" : [ " +
" { " +
" \"name\" : \"A\", " +
" \"percentage\" : 50, " +
" \"variableSet\" : {\"message\" : \"set A\"} " +
" }, " +
" { " +
" \"name\" : \"B\", " +
" \"percentage\" : 50, " +
" \"variableSet\" : {\"message\" : \"set B\"} " +
" } " +
" ]" +
"}");
KiiExperiment experiment = KiiExperiment.GetByID(id);
// verify Experiment
Assert.AreEqual("ID-001", experiment.ID);
Assert.AreEqual("Experiment for UT", experiment.Description);
Assert.AreEqual(2, experiment.Version);
Assert.AreEqual(KiiExperimentStatus.PAUSED, experiment.Status);
// verify VariationA
Variation variationA = experiment.GetVariationByName("A");
Assert.AreEqual("A", variationA.Name);
Assert.AreEqual(50, variationA.Percentage);
Assert.AreEqual(
new JsonObject("{\"message\" : \"set A\"}").ToString(),
variationA.VariableSet.ToString());
ConversionEvent eventA = experiment.GetConversionEventByName("Event-A");
Assert.AreEqual("Event-A", eventA.Name);
// verify VariationB
Variation variationB = experiment.GetVariationByName("B");
Assert.AreEqual("B", variationB.Name);
Assert.AreEqual(50, variationB.Percentage);
Assert.AreEqual(
new JsonObject("{\"message\" : \"set B\"}").ToString(),
variationB.VariableSet.ToString());
ConversionEvent eventB = experiment.GetConversionEventByName("Event-B");
Assert.AreEqual("Event-B", eventB.Name);
}
/// <summary>
/// Initializes a new instance of the <see cref="KiiCorp.Cloud.ABTesting.Variation"/> class.
/// </summary>
/// <remarks>
/// </remarks>
internal Variation(KiiExperiment parentExperiment, string name,
int percentage, JsonObject variableSet)
{
this.mParentExperimentID = parentExperiment.ID;
this.mParentExperimentVersion = parentExperiment.Version;
this.mParentExperimentConversionEvents = parentExperiment.ConversionEvents;
mParentExperimentStatus = parentExperiment.Status;
this.mName = name;
this.mPercentage = percentage;
this.mVariableSet = variableSet;
}
public void Test_0002_GetByID_With_InvalidID()
{
this.LogIn();
client.AddResponse(
200,
CreateKiiExperimentAsJsonString(id, description, version, status,
variationNameA, eventNameA, percentageA, variableSetA,
variationNameB, eventNameB, percentageB, variableSetB,
variationNameA)
);
KiiExperiment.GetByID("+100*");
}
public void Test_0008_GetAppliedVariation_Fallback()
{
this.LogIn();
client.AddResponse(
200,
CreateKiiExperimentAsJsonString(id, description, version, KiiExperimentStatus.DRAFT,
variationNameA, eventNameA, 0, variableSetA,
variationNameB, eventNameB, 100, variableSetB,
variationNameA)
);
KiiExperiment experiment = KiiExperiment.GetByID(id);
Variation variation = experiment.GetAppliedVariation(null);
Assert.IsNull(variation);
}
public void Test_0007_GetAppliedVariation()
{
this.LogIn();
client.AddResponse(
200,
CreateKiiExperimentAsJsonString(id, description, version, KiiExperimentStatus.RUNNING,
variationNameA, eventNameA, 0, variableSetA,
variationNameB, eventNameB, 100, variableSetB,
variationNameA)
);
KiiExperiment experiment = KiiExperiment.GetByID(id);
Variation variation = experiment.GetAppliedVariation(null);
Assert.AreEqual(experiment.Variations[1], variation);
}
public void Test_0008_OnCrank_With_Status_TERMINATED_Whit_ChosenVariation()
{
this.LogIn();
VariationSampler sampler = new VariationSamplerByKiiUser();
client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.TERMINATED, 50, 50, "B"));
KiiExperiment experiment = KiiExperiment.GetByID("000001");
Variation variationA = experiment.Variations[0]; // 50%
Variation variationB = experiment.Variations[1]; // 50%
Variation variation = sampler.ChooseVariation(experiment, null);
Assert.AreEqual(variationB, variation);
variation = sampler.ChooseVariation(experiment, variationA);
Assert.AreEqual(variationB, variation);
variation = sampler.ChooseVariation(experiment, variationB);
Assert.AreEqual(variationB, variation);
}
public void Test_0009_GetAppliedVariation_With_CustomSampler()
{
this.LogIn();
client.AddResponse(
200,
CreateKiiExperimentAsJsonString(id, description, version, KiiExperimentStatus.RUNNING,
variationNameA, eventNameA, 0, variableSetA,
variationNameB, eventNameB, 100, variableSetB,
variationNameA)
);
KiiExperiment experiment = KiiExperiment.GetByID(id);
Variation variation = experiment.GetAppliedVariation(null, new CustomVariationSampler());
// get VariationsA even its percentage is 0.
Assert.AreEqual(experiment.Variations[0], variation);
}
/// <summary>
/// Do sampling. Returns random variation based on the percentage configured in developer portal.
/// If sampling is failed, returns the fallback.
/// If the experiment has terminated and fixed variation has chosen, returns chosen variation.
/// Returned variation is same as <see cref="KiiExperiment.ChosenVariation"/>
/// </summary>
/// <remarks>
/// </remarks>
/// <returns>Applied variation for this time.</returns>
/// <param name="experiment">that requires sampling.</param>
/// <param name="fallback">The variation to return when failed to get the applied variation.</param>
public Variation ChooseVariation(KiiExperiment experiment, Variation fallback)
{
KiiUser user = KiiUser.CurrentUser;
if (user == null)
{
throw new InvalidOperationException(ErrorInfo.UTILS_NO_LOGIN);
}
switch (experiment.Status)
{
case KiiExperimentStatus.RUNNING:
int totalPercentage = 0;
foreach (Variation v in experiment.Variations)
{
totalPercentage += v.Percentage;
}
int random = new Random(user.Uri.GetHashCode()).Next(totalPercentage);
int i = 0;
int accumulated = 0;
foreach (Variation v in experiment.Variations)
{
if (random < accumulated + v.Percentage)
{
return(v);
}
accumulated += v.Percentage;
i++;
}
throw new InvalidOperationException("Unexpected error.");
case KiiExperimentStatus.DRAFT:
return(fallback);
case KiiExperimentStatus.PAUSED:
return(fallback);
case KiiExperimentStatus.TERMINATED:
if (experiment.ChosenVariation != null)
{
return(experiment.ChosenVariation);
}
return(fallback);
default:
throw new InvalidOperationException("Unknown status!");
}
}
/// <summary>
/// Creates instance of KiiExperiment from ApiResponse.
/// </summary>
/// <remarks>
/// </remarks>
/// <returns>The instance of KiiExperiment.</returns>
/// <param name="response">API response.</param>
private static KiiExperiment CreateFromResponse(ApiResponse response)
{
KiiExperiment experiment = null;
try
{
JsonObject json = new JsonObject(response.Body);
experiment = new KiiExperiment();
experiment.mId = json.GetString("_id");
experiment.mDescription = json.OptString("description", null);
experiment.mVersion = json.GetInt("version");
experiment.mStatus = ParseStatus(json);
// parse conversionEvents
JsonArray conversionEventsJson = json.GetJsonArray("conversionEvents");
List <ConversionEvent> conversionEventList = new List <ConversionEvent>();
for (int i = 0; i < conversionEventsJson.Length(); i++)
{
JsonObject conversionEventJson = conversionEventsJson.GetJsonObject(i);
conversionEventList.Add(new ConversionEvent(conversionEventJson.GetString("name")));
}
experiment.mConversionEvents = conversionEventList.ToArray();
// parse variations
JsonArray variationsJson = json.GetJsonArray("variations");
List <Variation> variationList = new List <Variation>();
for (int i = 0; i < variationsJson.Length(); i++)
{
JsonObject variationJson = variationsJson.GetJsonObject(i);
string varName = variationJson.GetString("name");
int percentage = variationJson.GetInt("percentage");
JsonObject test = variationJson.GetJsonObject("variableSet");
variationList.Add(new Variation(experiment, varName, percentage, test));
}
experiment.mVariations = variationList.ToArray();
string chosenVariationName = json.OptString("chosenVariation", null);
if (!Utils.IsEmpty(chosenVariationName))
{
experiment.mChosenVariation = experiment.GetVariationByName(chosenVariationName);
}
return(experiment);
}
catch (Exception e)
{
throw new IllegalKiiBaseObjectFormatException(e.Message);
}
}
public void Test_0007_OnCrank_With_Status_TERMINATED_Whitout_ChosenVariation()
{
System.Threading.Thread.Sleep(1);
VariationSampler sampler = new RandomVariationSampler();
client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.TERMINATED, 50, 50, null));
KiiExperiment experiment = KiiExperiment.GetByID("000001");
Variation variationA = experiment.Variations[0]; // 50%
Variation variationB = experiment.Variations[1]; // 50%
Variation variation = sampler.ChooseVariation(experiment, null);
Assert.IsNull(variation);
variation = sampler.ChooseVariation(experiment, variationA);
Assert.AreEqual(variationA, variation);
variation = sampler.ChooseVariation(experiment, variationB);
Assert.AreEqual(variationB, variation);
}
public void Test_0006_GetConversionEventByName()
{
this.LogIn();
client.AddResponse(
200,
CreateKiiExperimentAsJsonString(id, description, version, status,
variationNameA, eventNameA, percentageA, variableSetA,
variationNameB, eventNameB, percentageB, variableSetB,
variationNameA)
);
KiiExperiment experiment = KiiExperiment.GetByID(id);
ConversionEvent conversionEvent = experiment.GetConversionEventByName(eventNameB);
Assert.AreEqual(experiment.ConversionEvents [1], conversionEvent);
conversionEvent = experiment.GetConversionEventByName("hoge");
Assert.IsNull(conversionEvent);
}
/// <summary>
/// Do sampling. Returns random variation based on the percentage configured in developer portal.
/// If sampling is failed, returns the fallback.
/// If the experiment has terminated and fixed variation has chosen, returns chosen variation.
/// Returned variation is same as <see cref="KiiExperiment.ChosenVariation"/>
/// </summary>
/// <remarks>
/// </remarks>
/// <returns>Applied variation for this time.</returns>
/// <param name="experiment">that requires sampling.</param>
/// <param name="fallback">The variation to return when failed to get the applied variation.</param>
public Variation ChooseVariation(KiiExperiment experiment, Variation fallback)
{
switch (experiment.Status)
{
case KiiExperimentStatus.RUNNING:
int totalPercentage = 0;
foreach (Variation v in experiment.Variations)
{
totalPercentage += v.Percentage;
}
int random = 0;
lock (this)
{
random = this.mRandomGenerator.Next(totalPercentage);
}
int accumulated = 0;
foreach (Variation v in experiment.Variations)
{
accumulated += v.Percentage;
if (random < accumulated)
{
return(v);
}
}
throw new InvalidOperationException("Unexpected error.");
case KiiExperimentStatus.DRAFT:
return(fallback);
case KiiExperimentStatus.PAUSED:
return(fallback);
case KiiExperimentStatus.TERMINATED:
if (experiment.ChosenVariation != null)
{
return(experiment.ChosenVariation);
}
return(fallback);
default:
throw new InvalidOperationException("Unknown status!");
}
}
public void Test_0005_EventForConversionWhenStatusIsTerminated()
{
this.LogIn();
int experimentVersion = 10;
client.AddResponse(200, CreateKiiExperimentAsJsonString(experimentVersion, KiiExperimentStatus.TERMINATED, "UI-A", "EV-A", "UI-B", "EV-B"));
KiiExperiment experiment = KiiExperiment.GetByID("000001");
Variation variationA = experiment.Variations[0];
Variation variationB = experiment.Variations[1];
ConversionEvent eventA = experiment.ConversionEvents[0];
ConversionEvent eventB = experiment.ConversionEvents[1];
KiiEvent kiiEventA = variationA.EventForConversion(eventA);
Assert.IsInstanceOfType(typeof(KiiEvent.NullKiiEvent), kiiEventA);
KiiEvent kiiEventB = variationB.EventForConversion(eventB);
Assert.IsInstanceOfType(typeof(KiiEvent.NullKiiEvent), kiiEventB);
}
public void Test_0004_GetByID_ASync_With_InvalidID()
{
this.LogIn();
client.AddResponse(
200,
CreateKiiExperimentAsJsonString(id, description, version, status,
variationNameA, eventNameA, percentageA, variableSetA,
variationNameB, eventNameB, percentageB, variableSetB,
variationNameA)
);
KiiExperiment.GetByID("+100*", (KiiExperiment experiment, Exception e) => {
if (e == null)
{
Assert.Fail("ArgumentException was not thrown.");
}
if (e.GetType() != typeof(ArgumentException))
{
Assert.Fail("unexpected exception. " + e.GetType());
}
});
}
public void Test_0001_EventForConversion()
{
this.LogIn();
int experimentVersion = 10;
client.AddResponse(200, CreateKiiExperimentAsJsonString(experimentVersion, KiiExperimentStatus.RUNNING, "UI-A", "EV-A", "UI-B", "EV-B"));
KiiExperiment experiment = KiiExperiment.GetByID("000001");
Variation variationA = experiment.Variations[0];
Variation variationB = experiment.Variations[1];
ConversionEvent eventA = experiment.ConversionEvents[0];
ConversionEvent eventB = experiment.ConversionEvents[1];
KiiEvent kiiEventA = variationA.EventForConversion(0);
Assert.AreEqual(variationA.Name, kiiEventA["variationName"]);
Assert.AreEqual(eventA.Name, kiiEventA["conversionEvent"]);
Assert.AreEqual(experimentVersion, kiiEventA["version"]);
KiiEvent kiiEventB = variationB.EventForConversion(1);
Assert.AreEqual(variationB.Name, kiiEventB["variationName"]);
Assert.AreEqual(eventB.Name, kiiEventB["conversionEvent"]);
Assert.AreEqual(experimentVersion, kiiEventB["version"]);
}
public Variation ChooseVariation(KiiExperiment experiment, Variation fallback)
{
return(experiment.Variations[0]);
}
