public void Test_0001_OnCrank_Percentage_100_0()
 {
     this.LogIn();
     VariationSampler sampler = new VariationSamplerByKiiUser();
     client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.RUNNING, 100, 0, null));
     KiiExperiment experiment = KiiExperiment.GetByID("000001");
     Variation variationA = experiment.Variations[0]; // 100%
     Variation variationB = experiment.Variations[1]; // 0%
     bool selectedA = false;
     bool selectedB = false;
     for (int i = 0; i < 1000; i++)
     {
         this.LogIn();
         Variation variation = sampler.ChooseVariation(experiment, null);
         if (variation == variationA)
         {
             selectedA = true;
         }
         else if (variation == variationB)
         {
             selectedB = true;
         }
         else
         {
             Assert.Fail("sampler returned unexpected variation");
         }
     }
     Assert.IsTrue(selectedA);
     Assert.IsFalse(selectedB);
 }
 public void Test_0000_OnCrank_Percentage_50_50()
 {
     this.LogIn();
     VariationSampler sampler = new VariationSamplerByKiiUser();
     client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.RUNNING, 50, 50, null));
     KiiExperiment experiment = KiiExperiment.GetByID("000001");
     Variation variationA = experiment.Variations[0]; // 50%
     Variation variationB = experiment.Variations[1]; // 50%
     int selectedCountA = 0;
     int selectedCountB = 0;
     for (int i = 0; i < 1000; i++)
     {
         this.LogIn();
         Variation variation = sampler.ChooseVariation(experiment, null);
         if (variation == variationA)
         {
             selectedCountA++;
         }
         else if (variation == variationB)
         {
             selectedCountB++;
         }
         else
         {
             Assert.Fail("sampler returned unexpected variation");
         }
     }
     // This test will fail with a probability of 0.0017305361
     Assert.IsTrue(450 < selectedCountA, "selectedCountA=" + selectedCountA);
     Assert.IsTrue(550 > selectedCountA, "selectedCountA=" + selectedCountA);
     Assert.IsTrue(450 < selectedCountB, "selectedCountB=" + selectedCountB);
     Assert.IsTrue(550 > selectedCountB, "selectedCountB=" + selectedCountB);
 }
Exemple #3
0
        public void Test_0004_OnCrank_Percentage_95_5()
        {
            System.Threading.Thread.Sleep(1);
            VariationSampler sampler = new RandomVariationSampler();

            client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.RUNNING, 95, 5, null));
            KiiExperiment experiment     = KiiExperiment.GetByID("000001");
            Variation     variationA     = experiment.Variations[0]; // 95%
            Variation     variationB     = experiment.Variations[1]; // 5%
            int           selectedCountA = 0;
            int           selectedCountB = 0;

            for (int i = 0; i < 1000; i++)
            {
                System.Threading.Thread.Sleep(1);
                Variation variation = sampler.ChooseVariation(experiment, null);
                if (variation == variationA)
                {
                    selectedCountA++;
                }
                else if (variation == variationB)
                {
                    selectedCountB++;
                }
                else
                {
                    Assert.Fail("sampler returned unexpected variation");
                }
            }
            // This test will fail with a probability of 0.0349698573
            Assert.IsTrue(35 < selectedCountB, "selectedCountB=" + selectedCountB);
            Assert.IsTrue(65 > selectedCountB, "selectedCountB=" + selectedCountB);
        }
 /// <summary>
 /// Get the variation applied to this trial.
 /// Variation will be determined by specified rate of each variation in this experiment.
 /// Current login user information will be used for sampling.
 /// If the experiment has terminated with specified variant, the specified variant will be returned regardless of login user information.
 /// If the experiment has finished without specified variant, fallback will be returned.
 /// If the status of experiment is draft or paused, fallback will be returned.
 /// </summary>
 /// <remarks>
 /// </remarks>
 /// <returns>Applied variation for this trial.</returns>
 /// <param name="fallback">The variation to return when failed to get the applied variation. If you want to detect error, you need to set null.</param>
 public Variation GetAppliedVariation(Variation fallback)
 {
     if (KiiUser.CurrentUser == null)
     {
         return(fallback);
     }
     return(this.GetAppliedVariation(fallback, new VariationSamplerByKiiUser()));
 }
Exemple #5
0
        public void Test_0012_GetByID_Status_Is_Out_Of_Range()
        {
            this.LogIn();
            client.AddResponse(
                200,
                "{" +
                "    \"_id\" : \"ID-001\"," +
                "    \"description\" : \"Experiment for UT\"," +
                "    \"version\" : 2, " +
                "    \"status\" : 1000, " +
                "    \"conversionEvents\" : [ " +
                "        { \"name\" : \"Event-A\" }, " +
                "        { \"name\" : \"Event-B\" } " +
                "    ], " +
                "    \"variations\" : [ " +
                "        { " +
                "            \"name\" : \"A\", " +
                "            \"percentage\" : 50, " +
                "            \"variableSet\" : {\"message\" : \"set A\"} " +
                "        }, " +
                "        { " +
                "            \"name\" : \"B\", " +
                "            \"percentage\" : 50, " +
                "            \"variableSet\" : {\"message\" : \"set B\"} " +
                "        } " +
                "    ]" +
                "}");
            KiiExperiment experiment = KiiExperiment.GetByID(id);

            // verify Experiment
            Assert.AreEqual("ID-001", experiment.ID);
            Assert.AreEqual("Experiment for UT", experiment.Description);
            Assert.AreEqual(2, experiment.Version);
            Assert.AreEqual(KiiExperimentStatus.PAUSED, experiment.Status);

            // verify VariationA
            Variation variationA = experiment.GetVariationByName("A");

            Assert.AreEqual("A", variationA.Name);
            Assert.AreEqual(50, variationA.Percentage);
            Assert.AreEqual(
                new JsonObject("{\"message\" : \"set A\"}").ToString(),
                variationA.VariableSet.ToString());
            ConversionEvent eventA = experiment.GetConversionEventByName("Event-A");

            Assert.AreEqual("Event-A", eventA.Name);
            // verify VariationB
            Variation variationB = experiment.GetVariationByName("B");

            Assert.AreEqual("B", variationB.Name);
            Assert.AreEqual(50, variationB.Percentage);
            Assert.AreEqual(
                new JsonObject("{\"message\" : \"set B\"}").ToString(),
                variationB.VariableSet.ToString());
            ConversionEvent eventB = experiment.GetConversionEventByName("Event-B");

            Assert.AreEqual("Event-B", eventB.Name);
        }
Exemple #6
0
        public void Test_0007_GetAppliedVariation()
        {
            this.LogIn();
            client.AddResponse(
                200,
                CreateKiiExperimentAsJsonString(id, description, version, KiiExperimentStatus.RUNNING,
                                                variationNameA, eventNameA, 0, variableSetA,
                                                variationNameB, eventNameB, 100, variableSetB,
                                                variationNameA)
                );
            KiiExperiment experiment = KiiExperiment.GetByID(id);
            Variation     variation  = experiment.GetAppliedVariation(null);

            Assert.AreEqual(experiment.Variations[1], variation);
        }
 /// <summary>
 /// Get the variation applied to this trial.
 /// Sampler should return the variation according to the rate defined in this experiment.
 /// If you use <see cref="VariationSamplerByKiiUser"/> with current login user, It will be same as <see cref="KiiExperiment.GetAppliedVariation(Variation)"/>
 /// </summary>
 /// <remarks>
 /// </remarks>
 /// <returns>Applied variation for this trial.</returns>
 /// <param name="fallback">The variation to return when failed to get the applied variation. If you want to detect error, you need to set null.</param>
 /// <param name="sampler">Variation sampler.</param>
 public Variation GetAppliedVariation(Variation fallback, VariationSampler sampler)
 {
     if (sampler == null)
     {
         return(fallback);
     }
     try
     {
         return(sampler.ChooseVariation(this, fallback));
     }
     catch
     {
         return(fallback);
     }
 }
Exemple #8
0
        public void Test_0008_GetAppliedVariation_Fallback()
        {
            this.LogIn();
            client.AddResponse(
                200,
                CreateKiiExperimentAsJsonString(id, description, version, KiiExperimentStatus.DRAFT,
                                                variationNameA, eventNameA, 0, variableSetA,
                                                variationNameB, eventNameB, 100, variableSetB,
                                                variationNameA)
                );
            KiiExperiment experiment = KiiExperiment.GetByID(id);
            Variation     variation  = experiment.GetAppliedVariation(null);

            Assert.IsNull(variation);
        }
        public void Test_0008_OnCrank_With_Status_TERMINATED_Whit_ChosenVariation()
        {
            this.LogIn();
            VariationSampler sampler = new VariationSamplerByKiiUser();
            client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.TERMINATED, 50, 50, "B"));
            KiiExperiment experiment = KiiExperiment.GetByID("000001");
            Variation variationA = experiment.Variations[0]; // 50%
            Variation variationB = experiment.Variations[1]; // 50%

            Variation variation = sampler.ChooseVariation(experiment, null);
            Assert.AreEqual(variationB, variation);
            variation = sampler.ChooseVariation(experiment, variationA);
            Assert.AreEqual(variationB, variation);
            variation = sampler.ChooseVariation(experiment, variationB);
            Assert.AreEqual(variationB, variation);
        }
Exemple #10
0
        public void Test_0009_GetAppliedVariation_With_CustomSampler()
        {
            this.LogIn();
            client.AddResponse(
                200,
                CreateKiiExperimentAsJsonString(id, description, version, KiiExperimentStatus.RUNNING,
                                                variationNameA, eventNameA, 0, variableSetA,
                                                variationNameB, eventNameB, 100, variableSetB,
                                                variationNameA)
                );
            KiiExperiment experiment = KiiExperiment.GetByID(id);
            Variation     variation  = experiment.GetAppliedVariation(null, new CustomVariationSampler());

            // get VariationsA even its percentage is 0.
            Assert.AreEqual(experiment.Variations[0], variation);
        }
        /// <summary>
        /// Do sampling. Returns random variation based on the percentage configured in developer portal.
        /// If sampling is failed, returns the fallback.
        /// If the experiment has terminated and fixed variation has chosen, returns chosen variation.
        /// Returned variation is same as <see cref="KiiExperiment.ChosenVariation"/>
        /// </summary>
        /// <remarks>
        /// </remarks>
        /// <returns>Applied variation for this time.</returns>
        /// <param name="experiment">that requires sampling.</param>
        /// <param name="fallback">The variation to return when failed to get the applied variation.</param>
        public Variation ChooseVariation(KiiExperiment experiment, Variation fallback)
        {
            KiiUser user = KiiUser.CurrentUser;

            if (user == null)
            {
                throw new InvalidOperationException(ErrorInfo.UTILS_NO_LOGIN);
            }
            switch (experiment.Status)
            {
            case KiiExperimentStatus.RUNNING:
                int totalPercentage = 0;
                foreach (Variation v in experiment.Variations)
                {
                    totalPercentage += v.Percentage;
                }
                int random      = new Random(user.Uri.GetHashCode()).Next(totalPercentage);
                int i           = 0;
                int accumulated = 0;
                foreach (Variation v in experiment.Variations)
                {
                    if (random < accumulated + v.Percentage)
                    {
                        return(v);
                    }
                    accumulated += v.Percentage;
                    i++;
                }
                throw new InvalidOperationException("Unexpected error.");

            case KiiExperimentStatus.DRAFT:
                return(fallback);

            case KiiExperimentStatus.PAUSED:
                return(fallback);

            case KiiExperimentStatus.TERMINATED:
                if (experiment.ChosenVariation != null)
                {
                    return(experiment.ChosenVariation);
                }
                return(fallback);

            default:
                throw new InvalidOperationException("Unknown status!");
            }
        }
Exemple #12
0
        public void Test_0007_OnCrank_With_Status_TERMINATED_Whitout_ChosenVariation()
        {
            System.Threading.Thread.Sleep(1);
            VariationSampler sampler = new RandomVariationSampler();

            client.AddResponse(200, CreateKiiExperimentAsJsonString(KiiExperimentStatus.TERMINATED, 50, 50, null));
            KiiExperiment experiment = KiiExperiment.GetByID("000001");
            Variation     variationA = experiment.Variations[0]; // 50%
            Variation     variationB = experiment.Variations[1]; // 50%

            Variation variation = sampler.ChooseVariation(experiment, null);

            Assert.IsNull(variation);
            variation = sampler.ChooseVariation(experiment, variationA);
            Assert.AreEqual(variationA, variation);
            variation = sampler.ChooseVariation(experiment, variationB);
            Assert.AreEqual(variationB, variation);
        }
Exemple #13
0
        public void Test_0005_GetVariationByName()
        {
            this.LogIn();
            client.AddResponse(
                200,
                CreateKiiExperimentAsJsonString(id, description, version, status,
                                                variationNameA, eventNameA, percentageA, variableSetA,
                                                variationNameB, eventNameB, percentageB, variableSetB,
                                                variationNameA)
                );
            KiiExperiment experiment = KiiExperiment.GetByID(id);
            Variation     variation  = experiment.GetVariationByName(variationNameB);

            Assert.AreEqual(experiment.Variations[1], variation);

            variation = experiment.GetVariationByName("hoge");
            Assert.IsNull(variation);
        }
        /// <summary>
        /// Do sampling. Returns random variation based on the percentage configured in developer portal.
        /// If sampling is failed, returns the fallback.
        /// If the experiment has terminated and fixed variation has chosen, returns chosen variation.
        /// Returned variation is same as <see cref="KiiExperiment.ChosenVariation"/>
        /// </summary>
        /// <remarks>
        /// </remarks>
        /// <returns>Applied variation for this time.</returns>
        /// <param name="experiment">that requires sampling.</param>
        /// <param name="fallback">The variation to return when failed to get the applied variation.</param>
        public Variation ChooseVariation(KiiExperiment experiment, Variation fallback)
        {
            switch (experiment.Status)
            {
            case KiiExperimentStatus.RUNNING:
                int totalPercentage = 0;
                foreach (Variation v in experiment.Variations)
                {
                    totalPercentage += v.Percentage;
                }
                int random = 0;
                lock (this)
                {
                    random = this.mRandomGenerator.Next(totalPercentage);
                }
                int accumulated = 0;
                foreach (Variation v in experiment.Variations)
                {
                    accumulated += v.Percentage;
                    if (random < accumulated)
                    {
                        return(v);
                    }
                }
                throw new InvalidOperationException("Unexpected error.");

            case KiiExperimentStatus.DRAFT:
                return(fallback);

            case KiiExperimentStatus.PAUSED:
                return(fallback);

            case KiiExperimentStatus.TERMINATED:
                if (experiment.ChosenVariation != null)
                {
                    return(experiment.ChosenVariation);
                }
                return(fallback);

            default:
                throw new InvalidOperationException("Unknown status!");
            }
        }
        public void Test_0005_EventForConversionWhenStatusIsTerminated()
        {
            this.LogIn();
            int experimentVersion = 10;

            client.AddResponse(200, CreateKiiExperimentAsJsonString(experimentVersion, KiiExperimentStatus.TERMINATED, "UI-A", "EV-A", "UI-B", "EV-B"));
            KiiExperiment   experiment = KiiExperiment.GetByID("000001");
            Variation       variationA = experiment.Variations[0];
            Variation       variationB = experiment.Variations[1];
            ConversionEvent eventA     = experiment.ConversionEvents[0];
            ConversionEvent eventB     = experiment.ConversionEvents[1];

            KiiEvent kiiEventA = variationA.EventForConversion(eventA);

            Assert.IsInstanceOfType(typeof(KiiEvent.NullKiiEvent), kiiEventA);

            KiiEvent kiiEventB = variationB.EventForConversion(eventB);

            Assert.IsInstanceOfType(typeof(KiiEvent.NullKiiEvent), kiiEventB);
        }
        public void Test_0001_EventForConversion()
        {
            this.LogIn();
            int experimentVersion = 10;

            client.AddResponse(200, CreateKiiExperimentAsJsonString(experimentVersion, KiiExperimentStatus.RUNNING, "UI-A", "EV-A", "UI-B", "EV-B"));
            KiiExperiment   experiment = KiiExperiment.GetByID("000001");
            Variation       variationA = experiment.Variations[0];
            Variation       variationB = experiment.Variations[1];
            ConversionEvent eventA     = experiment.ConversionEvents[0];
            ConversionEvent eventB     = experiment.ConversionEvents[1];

            KiiEvent kiiEventA = variationA.EventForConversion(0);

            Assert.AreEqual(variationA.Name, kiiEventA["variationName"]);
            Assert.AreEqual(eventA.Name, kiiEventA["conversionEvent"]);
            Assert.AreEqual(experimentVersion, kiiEventA["version"]);

            KiiEvent kiiEventB = variationB.EventForConversion(1);

            Assert.AreEqual(variationB.Name, kiiEventB["variationName"]);
            Assert.AreEqual(eventB.Name, kiiEventB["conversionEvent"]);
            Assert.AreEqual(experimentVersion, kiiEventB["version"]);
        }
Exemple #17
0
 public Variation ChooseVariation(KiiExperiment experiment, Variation fallback)
 {
     return(experiment.Variations[0]);
 }