public void Test_0001_OnCrank_Percentage_100_0() { this.LogIn(); VariationSampler sampler = new VariationSamplerByKiiUser(); client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.RUNNING, 100, 0, null)); KiiExperiment experiment = KiiExperiment.GetByID("000001"); Variation variationA = experiment.Variations[0]; // 100% Variation variationB = experiment.Variations[1]; // 0% bool selectedA = false; bool selectedB = false; for (int i = 0; i < 1000; i++) { this.LogIn(); Variation variation = sampler.ChooseVariation(experiment, null); if (variation == variationA) { selectedA = true; } else if (variation == variationB) { selectedB = true; } else { Assert.Fail("sampler returned unexpected variation"); } } Assert.IsTrue(selectedA); Assert.IsFalse(selectedB); }
public void Test_0000_OnCrank_Percentage_50_50() { this.LogIn(); VariationSampler sampler = new VariationSamplerByKiiUser(); client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.RUNNING, 50, 50, null)); KiiExperiment experiment = KiiExperiment.GetByID("000001"); Variation variationA = experiment.Variations[0]; // 50% Variation variationB = experiment.Variations[1]; // 50% int selectedCountA = 0; int selectedCountB = 0; for (int i = 0; i < 1000; i++) { this.LogIn(); Variation variation = sampler.ChooseVariation(experiment, null); if (variation == variationA) { selectedCountA++; } else if (variation == variationB) { selectedCountB++; } else { Assert.Fail("sampler returned unexpected variation"); } } // This test will fail with a probability of 0.0017305361 Assert.IsTrue(450 < selectedCountA, "selectedCountA=" + selectedCountA); Assert.IsTrue(550 > selectedCountA, "selectedCountA=" + selectedCountA); Assert.IsTrue(450 < selectedCountB, "selectedCountB=" + selectedCountB); Assert.IsTrue(550 > selectedCountB, "selectedCountB=" + selectedCountB); }
public void Test_0004_OnCrank_Percentage_95_5() { System.Threading.Thread.Sleep(1); VariationSampler sampler = new RandomVariationSampler(); client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.RUNNING, 95, 5, null)); KiiExperiment experiment = KiiExperiment.GetByID("000001"); Variation variationA = experiment.Variations[0]; // 95% Variation variationB = experiment.Variations[1]; // 5% int selectedCountA = 0; int selectedCountB = 0; for (int i = 0; i < 1000; i++) { System.Threading.Thread.Sleep(1); Variation variation = sampler.ChooseVariation(experiment, null); if (variation == variationA) { selectedCountA++; } else if (variation == variationB) { selectedCountB++; } else { Assert.Fail("sampler returned unexpected variation"); } } // This test will fail with a probability of 0.0349698573 Assert.IsTrue(35 < selectedCountB, "selectedCountB=" + selectedCountB); Assert.IsTrue(65 > selectedCountB, "selectedCountB=" + selectedCountB); }
/// <summary> /// Get the variation applied to this trial. /// Variation will be determined by specified rate of each variation in this experiment. /// Current login user information will be used for sampling. /// If the experiment has terminated with specified variant, the specified variant will be returned regardless of login user information. /// If the experiment has finished without specified variant, fallback will be returned. /// If the status of experiment is draft or paused, fallback will be returned. /// </summary> /// <remarks> /// </remarks> /// <returns>Applied variation for this trial.</returns> /// <param name="fallback">The variation to return when failed to get the applied variation. If you want to detect error, you need to set null.</param> public Variation GetAppliedVariation(Variation fallback) { if (KiiUser.CurrentUser == null) { return(fallback); } return(this.GetAppliedVariation(fallback, new VariationSamplerByKiiUser())); }
public void Test_0012_GetByID_Status_Is_Out_Of_Range() { this.LogIn(); client.AddResponse( 200, "{" + " \"_id\" : \"ID-001\"," + " \"description\" : \"Experiment for UT\"," + " \"version\" : 2, " + " \"status\" : 1000, " + " \"conversionEvents\" : [ " + " { \"name\" : \"Event-A\" }, " + " { \"name\" : \"Event-B\" } " + " ], " + " \"variations\" : [ " + " { " + " \"name\" : \"A\", " + " \"percentage\" : 50, " + " \"variableSet\" : {\"message\" : \"set A\"} " + " }, " + " { " + " \"name\" : \"B\", " + " \"percentage\" : 50, " + " \"variableSet\" : {\"message\" : \"set B\"} " + " } " + " ]" + "}"); KiiExperiment experiment = KiiExperiment.GetByID(id); // verify Experiment Assert.AreEqual("ID-001", experiment.ID); Assert.AreEqual("Experiment for UT", experiment.Description); Assert.AreEqual(2, experiment.Version); Assert.AreEqual(KiiExperimentStatus.PAUSED, experiment.Status); // verify VariationA Variation variationA = experiment.GetVariationByName("A"); Assert.AreEqual("A", variationA.Name); Assert.AreEqual(50, variationA.Percentage); Assert.AreEqual( new JsonObject("{\"message\" : \"set A\"}").ToString(), variationA.VariableSet.ToString()); ConversionEvent eventA = experiment.GetConversionEventByName("Event-A"); Assert.AreEqual("Event-A", eventA.Name); // verify VariationB Variation variationB = experiment.GetVariationByName("B"); Assert.AreEqual("B", variationB.Name); Assert.AreEqual(50, variationB.Percentage); Assert.AreEqual( new JsonObject("{\"message\" : \"set B\"}").ToString(), variationB.VariableSet.ToString()); ConversionEvent eventB = experiment.GetConversionEventByName("Event-B"); Assert.AreEqual("Event-B", eventB.Name); }
public void Test_0007_GetAppliedVariation() { this.LogIn(); client.AddResponse( 200, CreateKiiExperimentAsJsonString(id, description, version, KiiExperimentStatus.RUNNING, variationNameA, eventNameA, 0, variableSetA, variationNameB, eventNameB, 100, variableSetB, variationNameA) ); KiiExperiment experiment = KiiExperiment.GetByID(id); Variation variation = experiment.GetAppliedVariation(null); Assert.AreEqual(experiment.Variations[1], variation); }
/// <summary> /// Get the variation applied to this trial. /// Sampler should return the variation according to the rate defined in this experiment. /// If you use <see cref="VariationSamplerByKiiUser"/> with current login user, It will be same as <see cref="KiiExperiment.GetAppliedVariation(Variation)"/> /// </summary> /// <remarks> /// </remarks> /// <returns>Applied variation for this trial.</returns> /// <param name="fallback">The variation to return when failed to get the applied variation. If you want to detect error, you need to set null.</param> /// <param name="sampler">Variation sampler.</param> public Variation GetAppliedVariation(Variation fallback, VariationSampler sampler) { if (sampler == null) { return(fallback); } try { return(sampler.ChooseVariation(this, fallback)); } catch { return(fallback); } }
public void Test_0008_GetAppliedVariation_Fallback() { this.LogIn(); client.AddResponse( 200, CreateKiiExperimentAsJsonString(id, description, version, KiiExperimentStatus.DRAFT, variationNameA, eventNameA, 0, variableSetA, variationNameB, eventNameB, 100, variableSetB, variationNameA) ); KiiExperiment experiment = KiiExperiment.GetByID(id); Variation variation = experiment.GetAppliedVariation(null); Assert.IsNull(variation); }
public void Test_0008_OnCrank_With_Status_TERMINATED_Whit_ChosenVariation() { this.LogIn(); VariationSampler sampler = new VariationSamplerByKiiUser(); client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.TERMINATED, 50, 50, "B")); KiiExperiment experiment = KiiExperiment.GetByID("000001"); Variation variationA = experiment.Variations[0]; // 50% Variation variationB = experiment.Variations[1]; // 50% Variation variation = sampler.ChooseVariation(experiment, null); Assert.AreEqual(variationB, variation); variation = sampler.ChooseVariation(experiment, variationA); Assert.AreEqual(variationB, variation); variation = sampler.ChooseVariation(experiment, variationB); Assert.AreEqual(variationB, variation); }
public void Test_0009_GetAppliedVariation_With_CustomSampler() { this.LogIn(); client.AddResponse( 200, CreateKiiExperimentAsJsonString(id, description, version, KiiExperimentStatus.RUNNING, variationNameA, eventNameA, 0, variableSetA, variationNameB, eventNameB, 100, variableSetB, variationNameA) ); KiiExperiment experiment = KiiExperiment.GetByID(id); Variation variation = experiment.GetAppliedVariation(null, new CustomVariationSampler()); // get VariationsA even its percentage is 0. Assert.AreEqual(experiment.Variations[0], variation); }
/// <summary> /// Do sampling. Returns random variation based on the percentage configured in developer portal. /// If sampling is failed, returns the fallback. /// If the experiment has terminated and fixed variation has chosen, returns chosen variation. /// Returned variation is same as <see cref="KiiExperiment.ChosenVariation"/> /// </summary> /// <remarks> /// </remarks> /// <returns>Applied variation for this time.</returns> /// <param name="experiment">that requires sampling.</param> /// <param name="fallback">The variation to return when failed to get the applied variation.</param> public Variation ChooseVariation(KiiExperiment experiment, Variation fallback) { KiiUser user = KiiUser.CurrentUser; if (user == null) { throw new InvalidOperationException(ErrorInfo.UTILS_NO_LOGIN); } switch (experiment.Status) { case KiiExperimentStatus.RUNNING: int totalPercentage = 0; foreach (Variation v in experiment.Variations) { totalPercentage += v.Percentage; } int random = new Random(user.Uri.GetHashCode()).Next(totalPercentage); int i = 0; int accumulated = 0; foreach (Variation v in experiment.Variations) { if (random < accumulated + v.Percentage) { return(v); } accumulated += v.Percentage; i++; } throw new InvalidOperationException("Unexpected error."); case KiiExperimentStatus.DRAFT: return(fallback); case KiiExperimentStatus.PAUSED: return(fallback); case KiiExperimentStatus.TERMINATED: if (experiment.ChosenVariation != null) { return(experiment.ChosenVariation); } return(fallback); default: throw new InvalidOperationException("Unknown status!"); } }
public void Test_0007_OnCrank_With_Status_TERMINATED_Whitout_ChosenVariation() { System.Threading.Thread.Sleep(1); VariationSampler sampler = new RandomVariationSampler(); client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.TERMINATED, 50, 50, null)); KiiExperiment experiment = KiiExperiment.GetByID("000001"); Variation variationA = experiment.Variations[0]; // 50% Variation variationB = experiment.Variations[1]; // 50% Variation variation = sampler.ChooseVariation(experiment, null); Assert.IsNull(variation); variation = sampler.ChooseVariation(experiment, variationA); Assert.AreEqual(variationA, variation); variation = sampler.ChooseVariation(experiment, variationB); Assert.AreEqual(variationB, variation); }
public void Test_0005_GetVariationByName() { this.LogIn(); client.AddResponse( 200, CreateKiiExperimentAsJsonString(id, description, version, status, variationNameA, eventNameA, percentageA, variableSetA, variationNameB, eventNameB, percentageB, variableSetB, variationNameA) ); KiiExperiment experiment = KiiExperiment.GetByID(id); Variation variation = experiment.GetVariationByName(variationNameB); Assert.AreEqual(experiment.Variations[1], variation); variation = experiment.GetVariationByName("hoge"); Assert.IsNull(variation); }
/// <summary> /// Do sampling. Returns random variation based on the percentage configured in developer portal. /// If sampling is failed, returns the fallback. /// If the experiment has terminated and fixed variation has chosen, returns chosen variation. /// Returned variation is same as <see cref="KiiExperiment.ChosenVariation"/> /// </summary> /// <remarks> /// </remarks> /// <returns>Applied variation for this time.</returns> /// <param name="experiment">that requires sampling.</param> /// <param name="fallback">The variation to return when failed to get the applied variation.</param> public Variation ChooseVariation(KiiExperiment experiment, Variation fallback) { switch (experiment.Status) { case KiiExperimentStatus.RUNNING: int totalPercentage = 0; foreach (Variation v in experiment.Variations) { totalPercentage += v.Percentage; } int random = 0; lock (this) { random = this.mRandomGenerator.Next(totalPercentage); } int accumulated = 0; foreach (Variation v in experiment.Variations) { accumulated += v.Percentage; if (random < accumulated) { return(v); } } throw new InvalidOperationException("Unexpected error."); case KiiExperimentStatus.DRAFT: return(fallback); case KiiExperimentStatus.PAUSED: return(fallback); case KiiExperimentStatus.TERMINATED: if (experiment.ChosenVariation != null) { return(experiment.ChosenVariation); } return(fallback); default: throw new InvalidOperationException("Unknown status!"); } }
public void Test_0005_EventForConversionWhenStatusIsTerminated() { this.LogIn(); int experimentVersion = 10; client.AddResponse(200, CreateKiiExperimentAsJsonString(experimentVersion, KiiExperimentStatus.TERMINATED, "UI-A", "EV-A", "UI-B", "EV-B")); KiiExperiment experiment = KiiExperiment.GetByID("000001"); Variation variationA = experiment.Variations[0]; Variation variationB = experiment.Variations[1]; ConversionEvent eventA = experiment.ConversionEvents[0]; ConversionEvent eventB = experiment.ConversionEvents[1]; KiiEvent kiiEventA = variationA.EventForConversion(eventA); Assert.IsInstanceOfType(typeof(KiiEvent.NullKiiEvent), kiiEventA); KiiEvent kiiEventB = variationB.EventForConversion(eventB); Assert.IsInstanceOfType(typeof(KiiEvent.NullKiiEvent), kiiEventB); }
public void Test_0001_EventForConversion() { this.LogIn(); int experimentVersion = 10; client.AddResponse(200, CreateKiiExperimentAsJsonString(experimentVersion, KiiExperimentStatus.RUNNING, "UI-A", "EV-A", "UI-B", "EV-B")); KiiExperiment experiment = KiiExperiment.GetByID("000001"); Variation variationA = experiment.Variations[0]; Variation variationB = experiment.Variations[1]; ConversionEvent eventA = experiment.ConversionEvents[0]; ConversionEvent eventB = experiment.ConversionEvents[1]; KiiEvent kiiEventA = variationA.EventForConversion(0); Assert.AreEqual(variationA.Name, kiiEventA["variationName"]); Assert.AreEqual(eventA.Name, kiiEventA["conversionEvent"]); Assert.AreEqual(experimentVersion, kiiEventA["version"]); KiiEvent kiiEventB = variationB.EventForConversion(1); Assert.AreEqual(variationB.Name, kiiEventB["variationName"]); Assert.AreEqual(eventB.Name, kiiEventB["conversionEvent"]); Assert.AreEqual(experimentVersion, kiiEventB["version"]); }
public Variation ChooseVariation(KiiExperiment experiment, Variation fallback) { return(experiment.Variations[0]); }