public void Test_0000_OnCrank_Percentage_50_50() { this.LogIn(); VariationSampler sampler = new VariationSamplerByKiiUser(); client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.RUNNING, 50, 50, null)); KiiExperiment experiment = KiiExperiment.GetByID("000001"); Variation variationA = experiment.Variations[0]; // 50% Variation variationB = experiment.Variations[1]; // 50% int selectedCountA = 0; int selectedCountB = 0; for (int i = 0; i < 1000; i++) { this.LogIn(); Variation variation = sampler.ChooseVariation(experiment, null); if (variation == variationA) { selectedCountA++; } else if (variation == variationB) { selectedCountB++; } else { Assert.Fail("sampler returned unexpected variation"); } } // This test will fail with a probability of 0.0017305361 Assert.IsTrue(450 < selectedCountA, "selectedCountA=" + selectedCountA); Assert.IsTrue(550 > selectedCountA, "selectedCountA=" + selectedCountA); Assert.IsTrue(450 < selectedCountB, "selectedCountB=" + selectedCountB); Assert.IsTrue(550 > selectedCountB, "selectedCountB=" + selectedCountB); }
public void Test_0004_OnCrank_Percentage_95_5() { System.Threading.Thread.Sleep(1); VariationSampler sampler = new RandomVariationSampler(); client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.RUNNING, 95, 5, null)); KiiExperiment experiment = KiiExperiment.GetByID("000001"); Variation variationA = experiment.Variations[0]; // 95% Variation variationB = experiment.Variations[1]; // 5% int selectedCountA = 0; int selectedCountB = 0; for (int i = 0; i < 1000; i++) { System.Threading.Thread.Sleep(1); Variation variation = sampler.ChooseVariation(experiment, null); if (variation == variationA) { selectedCountA++; } else if (variation == variationB) { selectedCountB++; } else { Assert.Fail("sampler returned unexpected variation"); } } // This test will fail with a probability of 0.0349698573 Assert.IsTrue(35 < selectedCountB, "selectedCountB=" + selectedCountB); Assert.IsTrue(65 > selectedCountB, "selectedCountB=" + selectedCountB); }
/// <summary> /// Get the experiment having the specified id in background. /// </summary> /// <remarks> /// NOTE: This api access to server. Should not be executed in UI/Main thread. /// </remarks> /// <returns>Experiment which has the specified id.</returns> /// <param name="experimentID">Experiment id.</param> /// <param name="callback">Called on completion of get experiment.</param> public static void GetByID(String experimentID, KiiExperimentCallback callback) { if (!IsValidExperimentID(experimentID)) { callback(null, new ArgumentException("Experiment id is invalid")); return; } Utils.CheckInitialize(false); string getUrl = Utils.Path(Kii.BaseUrl, "apps", Kii.AppId, "buckets", "_experiments", "objects", experimentID); KiiHttpClient client = Kii.AsyncHttpClientFactory.Create(getUrl, Kii.AppId, Kii.AppKey, KiiHttpMethod.GET); KiiCloudEngine.SetAuthBearer(client); client.SendRequest((ApiResponse response, Exception e) => { if (e != null) { callback(null, e); return; } else { try { KiiExperiment kiiExperiment = CreateFromResponse(response); callback(kiiExperiment, null); } catch (Exception ex) { callback(null, ex); } } }); }
public void Test_0001_OnCrank_Percentage_100_0() { this.LogIn(); VariationSampler sampler = new VariationSamplerByKiiUser(); client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.RUNNING, 100, 0, null)); KiiExperiment experiment = KiiExperiment.GetByID("000001"); Variation variationA = experiment.Variations[0]; // 100% Variation variationB = experiment.Variations[1]; // 0% bool selectedA = false; bool selectedB = false; for (int i = 0; i < 1000; i++) { this.LogIn(); Variation variation = sampler.ChooseVariation(experiment, null); if (variation == variationA) { selectedA = true; } else if (variation == variationB) { selectedB = true; } else { Assert.Fail("sampler returned unexpected variation"); } } Assert.IsTrue(selectedA); Assert.IsFalse(selectedB); }
public void Test_0001_GetByID() { this.LogIn(); client.AddResponse( 200, CreateKiiExperimentAsJsonString(id, description, version, status, variationNameA, eventNameA, percentageA, variableSetA, variationNameB, eventNameB, percentageB, variableSetB, variationNameA) ); KiiExperiment experiment = KiiExperiment.GetByID(id); // verify Experiment Assert.AreEqual(id, experiment.ID); Assert.AreEqual(description, experiment.Description); Assert.AreEqual(version, experiment.Version); Assert.AreEqual(status, experiment.Status); Assert.AreEqual(experiment.Variations[0], experiment.ChosenVariation); // verify VariationA Assert.AreEqual(variationNameA, experiment.Variations[0].Name); Assert.AreEqual(percentageA, experiment.Variations[0].Percentage); Assert.AreEqual(new JsonObject(variableSetA).ToString(), experiment.Variations[0].VariableSet.ToString()); Assert.AreEqual(eventNameA, experiment.ConversionEvents[0].Name); // verify VariationB Assert.AreEqual(variationNameB, experiment.Variations[1].Name); Assert.AreEqual(percentageB, experiment.Variations[1].Percentage); Assert.AreEqual(new JsonObject(variableSetB).ToString(), experiment.Variations[1].VariableSet.ToString()); Assert.AreEqual(eventNameB, experiment.ConversionEvents[1].Name); }
public void Test_0012_GetByID_Status_Is_Out_Of_Range() { this.LogIn(); client.AddResponse( 200, "{" + " \"_id\" : \"ID-001\"," + " \"description\" : \"Experiment for UT\"," + " \"version\" : 2, " + " \"status\" : 1000, " + " \"conversionEvents\" : [ " + " { \"name\" : \"Event-A\" }, " + " { \"name\" : \"Event-B\" } " + " ], " + " \"variations\" : [ " + " { " + " \"name\" : \"A\", " + " \"percentage\" : 50, " + " \"variableSet\" : {\"message\" : \"set A\"} " + " }, " + " { " + " \"name\" : \"B\", " + " \"percentage\" : 50, " + " \"variableSet\" : {\"message\" : \"set B\"} " + " } " + " ]" + "}"); KiiExperiment experiment = KiiExperiment.GetByID(id); // verify Experiment Assert.AreEqual("ID-001", experiment.ID); Assert.AreEqual("Experiment for UT", experiment.Description); Assert.AreEqual(2, experiment.Version); Assert.AreEqual(KiiExperimentStatus.PAUSED, experiment.Status); // verify VariationA Variation variationA = experiment.GetVariationByName("A"); Assert.AreEqual("A", variationA.Name); Assert.AreEqual(50, variationA.Percentage); Assert.AreEqual( new JsonObject("{\"message\" : \"set A\"}").ToString(), variationA.VariableSet.ToString()); ConversionEvent eventA = experiment.GetConversionEventByName("Event-A"); Assert.AreEqual("Event-A", eventA.Name); // verify VariationB Variation variationB = experiment.GetVariationByName("B"); Assert.AreEqual("B", variationB.Name); Assert.AreEqual(50, variationB.Percentage); Assert.AreEqual( new JsonObject("{\"message\" : \"set B\"}").ToString(), variationB.VariableSet.ToString()); ConversionEvent eventB = experiment.GetConversionEventByName("Event-B"); Assert.AreEqual("Event-B", eventB.Name); }
/// <summary> /// Initializes a new instance of the <see cref="KiiCorp.Cloud.ABTesting.Variation"/> class. /// </summary> /// <remarks> /// </remarks> internal Variation(KiiExperiment parentExperiment, string name, int percentage, JsonObject variableSet) { this.mParentExperimentID = parentExperiment.ID; this.mParentExperimentVersion = parentExperiment.Version; this.mParentExperimentConversionEvents = parentExperiment.ConversionEvents; mParentExperimentStatus = parentExperiment.Status; this.mName = name; this.mPercentage = percentage; this.mVariableSet = variableSet; }
public void Test_0002_GetByID_With_InvalidID() { this.LogIn(); client.AddResponse( 200, CreateKiiExperimentAsJsonString(id, description, version, status, variationNameA, eventNameA, percentageA, variableSetA, variationNameB, eventNameB, percentageB, variableSetB, variationNameA) ); KiiExperiment.GetByID("+100*"); }
public void Test_0008_GetAppliedVariation_Fallback() { this.LogIn(); client.AddResponse( 200, CreateKiiExperimentAsJsonString(id, description, version, KiiExperimentStatus.DRAFT, variationNameA, eventNameA, 0, variableSetA, variationNameB, eventNameB, 100, variableSetB, variationNameA) ); KiiExperiment experiment = KiiExperiment.GetByID(id); Variation variation = experiment.GetAppliedVariation(null); Assert.IsNull(variation); }
public void Test_0007_GetAppliedVariation() { this.LogIn(); client.AddResponse( 200, CreateKiiExperimentAsJsonString(id, description, version, KiiExperimentStatus.RUNNING, variationNameA, eventNameA, 0, variableSetA, variationNameB, eventNameB, 100, variableSetB, variationNameA) ); KiiExperiment experiment = KiiExperiment.GetByID(id); Variation variation = experiment.GetAppliedVariation(null); Assert.AreEqual(experiment.Variations[1], variation); }
public void Test_0008_OnCrank_With_Status_TERMINATED_Whit_ChosenVariation() { this.LogIn(); VariationSampler sampler = new VariationSamplerByKiiUser(); client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.TERMINATED, 50, 50, "B")); KiiExperiment experiment = KiiExperiment.GetByID("000001"); Variation variationA = experiment.Variations[0]; // 50% Variation variationB = experiment.Variations[1]; // 50% Variation variation = sampler.ChooseVariation(experiment, null); Assert.AreEqual(variationB, variation); variation = sampler.ChooseVariation(experiment, variationA); Assert.AreEqual(variationB, variation); variation = sampler.ChooseVariation(experiment, variationB); Assert.AreEqual(variationB, variation); }
public void Test_0009_GetAppliedVariation_With_CustomSampler() { this.LogIn(); client.AddResponse( 200, CreateKiiExperimentAsJsonString(id, description, version, KiiExperimentStatus.RUNNING, variationNameA, eventNameA, 0, variableSetA, variationNameB, eventNameB, 100, variableSetB, variationNameA) ); KiiExperiment experiment = KiiExperiment.GetByID(id); Variation variation = experiment.GetAppliedVariation(null, new CustomVariationSampler()); // get VariationsA even its percentage is 0. Assert.AreEqual(experiment.Variations[0], variation); }
/// <summary> /// Do sampling. Returns random variation based on the percentage configured in developer portal. /// If sampling is failed, returns the fallback. /// If the experiment has terminated and fixed variation has chosen, returns chosen variation. /// Returned variation is same as <see cref="KiiExperiment.ChosenVariation"/> /// </summary> /// <remarks> /// </remarks> /// <returns>Applied variation for this time.</returns> /// <param name="experiment">that requires sampling.</param> /// <param name="fallback">The variation to return when failed to get the applied variation.</param> public Variation ChooseVariation(KiiExperiment experiment, Variation fallback) { KiiUser user = KiiUser.CurrentUser; if (user == null) { throw new InvalidOperationException(ErrorInfo.UTILS_NO_LOGIN); } switch (experiment.Status) { case KiiExperimentStatus.RUNNING: int totalPercentage = 0; foreach (Variation v in experiment.Variations) { totalPercentage += v.Percentage; } int random = new Random(user.Uri.GetHashCode()).Next(totalPercentage); int i = 0; int accumulated = 0; foreach (Variation v in experiment.Variations) { if (random < accumulated + v.Percentage) { return(v); } accumulated += v.Percentage; i++; } throw new InvalidOperationException("Unexpected error."); case KiiExperimentStatus.DRAFT: return(fallback); case KiiExperimentStatus.PAUSED: return(fallback); case KiiExperimentStatus.TERMINATED: if (experiment.ChosenVariation != null) { return(experiment.ChosenVariation); } return(fallback); default: throw new InvalidOperationException("Unknown status!"); } }
/// <summary> /// Creates instance of KiiExperiment from ApiResponse. /// </summary> /// <remarks> /// </remarks> /// <returns>The instance of KiiExperiment.</returns> /// <param name="response">API response.</param> private static KiiExperiment CreateFromResponse(ApiResponse response) { KiiExperiment experiment = null; try { JsonObject json = new JsonObject(response.Body); experiment = new KiiExperiment(); experiment.mId = json.GetString("_id"); experiment.mDescription = json.OptString("description", null); experiment.mVersion = json.GetInt("version"); experiment.mStatus = ParseStatus(json); // parse conversionEvents JsonArray conversionEventsJson = json.GetJsonArray("conversionEvents"); List <ConversionEvent> conversionEventList = new List <ConversionEvent>(); for (int i = 0; i < conversionEventsJson.Length(); i++) { JsonObject conversionEventJson = conversionEventsJson.GetJsonObject(i); conversionEventList.Add(new ConversionEvent(conversionEventJson.GetString("name"))); } experiment.mConversionEvents = conversionEventList.ToArray(); // parse variations JsonArray variationsJson = json.GetJsonArray("variations"); List <Variation> variationList = new List <Variation>(); for (int i = 0; i < variationsJson.Length(); i++) { JsonObject variationJson = variationsJson.GetJsonObject(i); string varName = variationJson.GetString("name"); int percentage = variationJson.GetInt("percentage"); JsonObject test = variationJson.GetJsonObject("variableSet"); variationList.Add(new Variation(experiment, varName, percentage, test)); } experiment.mVariations = variationList.ToArray(); string chosenVariationName = json.OptString("chosenVariation", null); if (!Utils.IsEmpty(chosenVariationName)) { experiment.mChosenVariation = experiment.GetVariationByName(chosenVariationName); } return(experiment); } catch (Exception e) { throw new IllegalKiiBaseObjectFormatException(e.Message); } }
public void Test_0007_OnCrank_With_Status_TERMINATED_Whitout_ChosenVariation() { System.Threading.Thread.Sleep(1); VariationSampler sampler = new RandomVariationSampler(); client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.TERMINATED, 50, 50, null)); KiiExperiment experiment = KiiExperiment.GetByID("000001"); Variation variationA = experiment.Variations[0]; // 50% Variation variationB = experiment.Variations[1]; // 50% Variation variation = sampler.ChooseVariation(experiment, null); Assert.IsNull(variation); variation = sampler.ChooseVariation(experiment, variationA); Assert.AreEqual(variationA, variation); variation = sampler.ChooseVariation(experiment, variationB); Assert.AreEqual(variationB, variation); }
public void Test_0006_GetConversionEventByName() { this.LogIn(); client.AddResponse( 200, CreateKiiExperimentAsJsonString(id, description, version, status, variationNameA, eventNameA, percentageA, variableSetA, variationNameB, eventNameB, percentageB, variableSetB, variationNameA) ); KiiExperiment experiment = KiiExperiment.GetByID(id); ConversionEvent conversionEvent = experiment.GetConversionEventByName(eventNameB); Assert.AreEqual(experiment.ConversionEvents [1], conversionEvent); conversionEvent = experiment.GetConversionEventByName("hoge"); Assert.IsNull(conversionEvent); }
/// <summary> /// Do sampling. Returns random variation based on the percentage configured in developer portal. /// If sampling is failed, returns the fallback. /// If the experiment has terminated and fixed variation has chosen, returns chosen variation. /// Returned variation is same as <see cref="KiiExperiment.ChosenVariation"/> /// </summary> /// <remarks> /// </remarks> /// <returns>Applied variation for this time.</returns> /// <param name="experiment">that requires sampling.</param> /// <param name="fallback">The variation to return when failed to get the applied variation.</param> public Variation ChooseVariation(KiiExperiment experiment, Variation fallback) { switch (experiment.Status) { case KiiExperimentStatus.RUNNING: int totalPercentage = 0; foreach (Variation v in experiment.Variations) { totalPercentage += v.Percentage; } int random = 0; lock (this) { random = this.mRandomGenerator.Next(totalPercentage); } int accumulated = 0; foreach (Variation v in experiment.Variations) { accumulated += v.Percentage; if (random < accumulated) { return(v); } } throw new InvalidOperationException("Unexpected error."); case KiiExperimentStatus.DRAFT: return(fallback); case KiiExperimentStatus.PAUSED: return(fallback); case KiiExperimentStatus.TERMINATED: if (experiment.ChosenVariation != null) { return(experiment.ChosenVariation); } return(fallback); default: throw new InvalidOperationException("Unknown status!"); } }
public void Test_0005_EventForConversionWhenStatusIsTerminated() { this.LogIn(); int experimentVersion = 10; client.AddResponse(200, CreateKiiExperimentAsJsonString(experimentVersion, KiiExperimentStatus.TERMINATED, "UI-A", "EV-A", "UI-B", "EV-B")); KiiExperiment experiment = KiiExperiment.GetByID("000001"); Variation variationA = experiment.Variations[0]; Variation variationB = experiment.Variations[1]; ConversionEvent eventA = experiment.ConversionEvents[0]; ConversionEvent eventB = experiment.ConversionEvents[1]; KiiEvent kiiEventA = variationA.EventForConversion(eventA); Assert.IsInstanceOfType(typeof(KiiEvent.NullKiiEvent), kiiEventA); KiiEvent kiiEventB = variationB.EventForConversion(eventB); Assert.IsInstanceOfType(typeof(KiiEvent.NullKiiEvent), kiiEventB); }
public void Test_0004_GetByID_ASync_With_InvalidID() { this.LogIn(); client.AddResponse( 200, CreateKiiExperimentAsJsonString(id, description, version, status, variationNameA, eventNameA, percentageA, variableSetA, variationNameB, eventNameB, percentageB, variableSetB, variationNameA) ); KiiExperiment.GetByID("+100*", (KiiExperiment experiment, Exception e) => { if (e == null) { Assert.Fail("ArgumentException was not thrown."); } if (e.GetType() != typeof(ArgumentException)) { Assert.Fail("unexpected exception. " + e.GetType()); } }); }
public void Test_0001_EventForConversion() { this.LogIn(); int experimentVersion = 10; client.AddResponse(200, CreateKiiExperimentAsJsonString(experimentVersion, KiiExperimentStatus.RUNNING, "UI-A", "EV-A", "UI-B", "EV-B")); KiiExperiment experiment = KiiExperiment.GetByID("000001"); Variation variationA = experiment.Variations[0]; Variation variationB = experiment.Variations[1]; ConversionEvent eventA = experiment.ConversionEvents[0]; ConversionEvent eventB = experiment.ConversionEvents[1]; KiiEvent kiiEventA = variationA.EventForConversion(0); Assert.AreEqual(variationA.Name, kiiEventA["variationName"]); Assert.AreEqual(eventA.Name, kiiEventA["conversionEvent"]); Assert.AreEqual(experimentVersion, kiiEventA["version"]); KiiEvent kiiEventB = variationB.EventForConversion(1); Assert.AreEqual(variationB.Name, kiiEventB["variationName"]); Assert.AreEqual(eventB.Name, kiiEventB["conversionEvent"]); Assert.AreEqual(experimentVersion, kiiEventB["version"]); }
public Variation ChooseVariation(KiiExperiment experiment, Variation fallback) { return(experiment.Variations[0]); }