Example #1
0
        public void UpdateTestResult()
        {
            Result = MVTAnalyser.AnalyseTest(this);

            //Check if the test is reached significance
            //If any of the variant reached significance, the finalise test
            if (Result != null)
            {
                if (Result.IsAnyReachedSignificance || (AssignedGuidsTotal > Result.GuidLimitBeforeAbandon))
                {
                    FinishTest();
                }
            }
        }
Example #2
0
        public static MVTResult AnalyseTest(MultiVariateTest test)
        {
            MVTResult result   = null;
            int       variants = -1;
            int       GuidLimitBeforeAbandon;

            int[]         SampleSize;
            int[]         ConvertedSize;
            double[]      Means;
            List <byte>[] Values;
            double[]      StdDev;
            double[]      Variances;
            double[]      TScores;
            int[]         DFree;
            int[]         NextDFree;
            double[]      ReqPVal;
            bool[]        ReachedSignificance;


            variants = test.GetNumberOfVariants();
            if (variants < 1)
            {
                return(result);
            }

            SampleSize             = new int[variants];
            ConvertedSize          = new int[variants];
            Means                  = new double[variants];
            StdDev                 = new double[variants];
            Values                 = new List <byte> [variants];
            Variances              = new double[variants - 1];
            TScores                = new double[variants - 1];
            DFree                  = new int[variants - 1];
            NextDFree              = new int[variants - 1];
            ReqPVal                = new double[variants - 1];
            ReachedSignificance    = new bool[variants - 1];
            GuidLimitBeforeAbandon = Configuration.STRAT_AVG_DailyTraffic_Expected * Configuration.STRAT_MAX_MVT_RunDaysBeforeAbandon;


            //Getting sample sizes
            for (int i = 0; i < variants; i++)
            {
                SampleSize[i] = test.GetAssignedGuidNumber(i);

                if (Configuration.ANA_SmallSampleSizeOk == false && SampleSize[i] < Configuration.ANA_MinSampleSizePerVariant)
                {
                    return(null);
                }

                if (SampleSize[i] == 0)
                {
                    return(null);
                }
            }

            //Getting the conversion values as 0 or 1
            for (int i = 0; i < variants; i++)
            {
                Values[i] = test.GetVariantConversionValuesAsNumbers(i);

                if (Values[i] == null)
                {
                    return(result);
                }
            }

            //Getting means
            for (int i = 0; i < variants; i++)
            {
                ConvertedSize[i] = 0;
                foreach (byte b in Values[i])
                {
                    ConvertedSize[i] += b;
                }
                Means[i] = (double)ConvertedSize[i] / (double)SampleSize[i];
            }

            //Calculating STD devs
            for (int i = 0; i < variants; i++)
            {
                double sum = 0;

                for (int j = 0; j < SampleSize[i]; j++)
                {
                    double tmp = Values[i][j] - Means[i];
                    tmp *= tmp;
                    tmp /= (SampleSize[i] - 1);

                    sum += tmp;
                }

                StdDev[i] = Math.Sqrt(sum);
            }

            //Calculating variance between sample groups
            //Assuming 0 is control
            for (int i = 1; i < variants; i++)
            {
                double v = StdDev[0] / SampleSize[0];
                v += (StdDev[i] / SampleSize[i]);

                Variances[i - 1] = Math.Sqrt(v);
            }

            //Calculating T-scores
            //Assuming 0 is control
            for (int i = 1; i < variants; i++)
            {
                TScores[i - 1] = Math.Abs(((Means[0] - Means[i]) / Variances[i - 1]));
            }

            //Determine degree of freedom
            for (int i = 1; i < variants; i++)
            {
                DFree[i - 1] = SampleSize[0] + SampleSize[i] - 2;
            }

            //Find the next P value needs to be reached to get significance based on the current freedom
            //Also check if the variants reached significance
            for (int i = 1; i < variants; i++)
            {
                int j;
                for (j = 0; j < Configuration.ANA_D_Free.Length - 1; j++)
                {
                    if (DFree[i - 1] <= Configuration.ANA_D_Free[j])
                    {
                        break;
                    }
                }

                NextDFree[i - 1] = Configuration.ANA_D_Free[j];
                ReqPVal[i - 1]   = Configuration.ANA_P_Value[j];

                if (ReqPVal[i - 1] < TScores[i - 1])
                {
                    ReachedSignificance[i - 1] = true;
                }
            }



            //Setting up results to return
            result                          = new MVTResult();
            result.Means                    = Means;
            result.TScores                  = TScores;
            result.DFree                    = DFree;
            result.NextDFree                = NextDFree;
            result.ReqPVal                  = ReqPVal;
            result.ReachedSignificance      = ReachedSignificance;
            result.IsAnyReachedSignificance = ReachedSignificance.Contains(true);
            result.SampleSize               = SampleSize;
            result.ConvertedSize            = ConvertedSize;
            result.Variants                 = variants;
            result.STDevs                   = StdDev;
            result.GuidLimitBeforeAbandon   = GuidLimitBeforeAbandon;

            return(result);
        }
Example #3
0
        public static void PrintDashboard(CommandLineInterface _CLI)
        {
            int TotalMVTs        = MVTHistory.GetMVTCount(false);
            int LiveMVTs         = MVTHistory.GetMVTCount(true);
            int CompletedMVTs    = TotalMVTs - LiveMVTs;
            int PositiveMVTs     = MVTHistory.GetMVTCount(TestResultType.Positive);
            int NegativeMVTs     = MVTHistory.GetMVTCount(TestResultType.Negative);
            int InconclusiveMVTs = MVTHistory.GetMVTCount(TestResultType.Inconclusive);

            MultiVariateTest[] LiveTests = MVTHistory.GetLiveTests();

            string result = "************************** OVERALL TESTING SUMMARY *************************\n\n";

            result += "Total MVTs launched: " + TotalMVTs.ToString()
                      + " (" + CompletedMVTs.ToString() + " completed, "
                      + LiveMVTs.ToString() + " currently live)\n";

            result += "Test results:\t" + PositiveMVTs.ToString() + " Positive\t"
                      + NegativeMVTs.ToString() + " Negative\t"
                      + InconclusiveMVTs.ToString() + " Inconclusive (abandoned) \n\n";

            if (LiveTests != null && LiveTests.Length > 0)
            {
                result += "******************************** LIVE TESTS ********************************\n";

                for (int i = 0; i < LiveTests.Length; i++)
                {
                    int           testId  = LiveTests[i].Id;
                    TestTypesEnum type    = LiveTests[i].TestType;
                    string        typeStr = "";
                    if (type == TestTypesEnum.ADDTOCARTBUTTON)
                    {
                        typeStr = "Add to Cart Btn";
                    }

                    DateTime?launched       = LiveTests[i].GetLaunchDateTime;
                    int      variantsNumber = LiveTests[i].GetNumberOfVariants();

                    MVTResult TResult = LiveTests[i].GetResult;



                    result += "[MVT ID: " + testId.ToString() + "]  [Type: " + typeStr
                              + "]  [Launch date: " + launched.ToString() + "]\n";

                    result += "[Variants: " + variantsNumber.ToString() + " (including control)]\n";


                    //Displaying ctr and test variant change details
                    //Only "Add to cart" A/B test display supported currently.
                    if (variantsNumber == 2 && type == TestTypesEnum.ADDTOCARTBUTTON)
                    {
                        AddToCartButtonVariation ctrVar = (AddToCartButtonVariation)LiveTests[i].GetVariant(0).GetVariationType();
                        AddToCartButtonVariation tstVar = (AddToCartButtonVariation)LiveTests[i].GetVariant(1).GetVariationType();

                        result += "[Control: ";
                        if (ctrVar.colorPalette == null && ctrVar.addToCartTextCopies == null)
                        {
                            result += "No changes";
                        }
                        else
                        {
                            if (ctrVar.colorPalette != null)
                            {
                                result += "Colour: " + ctrVar.colorPalette + "   ";
                            }

                            if (ctrVar.addToCartTextCopies != null)
                            {
                                result += "Copy change: " + ctrVar.addToCartTextCopies;
                            }
                        }
                        result += "]\n";

                        result += "[Test: ";
                        if (tstVar.colorPalette == null && tstVar.addToCartTextCopies == null)
                        {
                            result += "No changes";
                        }
                        else
                        {
                            if (tstVar.colorPalette != null)
                            {
                                result += "Colour: " + tstVar.colorPalette + "   ";
                            }

                            if (tstVar.addToCartTextCopies != null)
                            {
                                result += "Copy change: " + tstVar.addToCartTextCopies;
                            }
                        }
                        result += "]\n\n";
                    }

                    //Displaying ctr and test variant visitors, cvr and stdev. Works for A/B only.
                    if (TResult != null && variantsNumber == 2)
                    {
                        if (TResult.SampleSize.Length > 1 &&
                            TResult.Means.Length > 1 &&
                            TResult.ConvertedSize.Length > 1 &&
                            TResult.STDevs.Length > 1)
                        {
                            result += "\t[Visitors]\t[Converted]\t[%]\t[Std Dev]\n";

                            result += "CTRL\t" + TResult.SampleSize[0].ToString() + "\t\t"
                                      + TResult.ConvertedSize[0].ToString() + "\t\t"
                                      + TruncString((TResult.Means[0] * 100).ToString(), 4) + "%\t"
                                      + TruncString(TResult.STDevs[0].ToString(), 5) + "\n";



                            result += "TEST\t" + TResult.SampleSize[1].ToString() + "\t\t"
                                      + TResult.ConvertedSize[1].ToString() + "\t\t"
                                      + TruncString((TResult.Means[1] * 100).ToString(), 4) + "%\t"
                                      + TruncString(TResult.STDevs[1].ToString(), 5) + "\n\n";

                            result += "[T-score: " + TruncString(TResult.TScores[0].ToString(), 6) + "]  "
                                      + "[Degree of freedom: " + TResult.DFree[0].ToString()
                                      + " (next known is " + TResult.NextDFree[0].ToString() + ")]\n";

                            result += "[Required P for next Degree of Freedom: " + TruncString(TResult.ReqPVal[0].ToString(), 6) + "]\n";
                        }
                    }

                    result += "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~";
                }
            }
            else
            {
                //No live tests found, check if all completed.
                result += "\n No live tests running currently.\n";

                if (AddToCartButton.IsAllVariationTested())
                {
                    result += "All the 'Add To Cart Button' type variants have been tested. The best performing variant is live on 100%.";
                }
            }

            _CLI.Out(result, false, false);
        }
Example #4
0
        public static void PrintStats(CommandLineInterface _CLI)
        {
            string result = "\n\n********** MVT STATS **********\n\n";


            int TotalMVTs = MVTHistory.GetMVTCount(false);
            int LiveMVTs  = MVTHistory.GetMVTCount(true);

            result += "Total MVTs: " + TotalMVTs.ToString() + "\n";
            result += "Live MVTs: " + LiveMVTs.ToString() + "\n";

            if (LiveMVTs == 0)
            {
                result += "\n\n********** MVT STATS **********\n\n";
                _CLI.Out(result, false, false);
                return;
            }

            MultiVariateTest[] LiveTests = MVTHistory.GetLiveTests();

            if (LiveTests == null || LiveTests.Length == 0)
            {
                result += "\n\n********** MVT STATS **********\n\n";

                //No live tests found, check if all completed.
                result += "\n No live tests running currently.\n";
                if (AddToCartButton.IsAllVariationTested())
                {
                    result += "All the 'Add To Cart Button' type variants have been tested. The best performing variant is live on 100%.";
                }

                _CLI.Out(result, false, false);
                return;
            }



            for (int i = 0; i < LiveTests.Length; i++)
            {
                result += "\n----\n";

                result += "MVT ID: " + LiveTests[i].Id.ToString() + "\n";
                result += "Variant number (including control): " + LiveTests[i].GetNumberOfVariants().ToString() + "\n";
                result += "Launch date: " + LiveTests[i].GetLaunchDateTime.ToString() + "\n\n";

                MVTResult testResult = LiveTests[i].GetResult;
                if (testResult != null)
                {
                    result += "Variants (including control): " + testResult.Variants.ToString() + "\n";
                    result += "Any variant reached significance: " + testResult.ReachedSignificance.ToString() + "\n";


                    result += "Visitors: ";
                    for (int j = 0; j < testResult.Variants; j++)
                    {
                        result += "[" + j.ToString() + "]: " + testResult.SampleSize[j] + "\t";
                    }
                    result += "\n";

                    result += "Converted visitors: ";
                    for (int j = 0; j < testResult.Variants; j++)
                    {
                        result += "[" + j.ToString() + "]: " + testResult.ConvertedSize[j] + "\t";
                    }
                    result += "\n";

                    result += "Means: ";
                    for (int j = 0; j < testResult.Variants; j++)
                    {
                        result += "[" + j.ToString() + "]: " + testResult.Means[j] + "\t";
                    }
                    result += "\n";

                    result += "Standard Deviations: ";
                    for (int j = 0; j < testResult.Variants; j++)
                    {
                        result += "[" + j.ToString() + "]: " + testResult.STDevs[j] + "\t";
                    }
                    result += "\n\n";

                    result += "T-Score: ";
                    for (int j = 0; j < testResult.Variants - 1; j++)
                    {
                        result += "[" + j.ToString() + "]: " + testResult.TScores[j] + "\t";
                    }
                    result += "\n";

                    result += "Degree of Freedom: ";
                    for (int j = 0; j < testResult.Variants - 1; j++)
                    {
                        result += "[" + j.ToString() + "]: " + testResult.DFree[j] + "\t";
                    }
                    result += "\n";

                    result += "Next known Degree Freedom: ";
                    for (int j = 0; j < testResult.Variants - 1; j++)
                    {
                        result += "[" + j.ToString() + "]: " + testResult.NextDFree[j] + "\t";
                    }
                    result += "\n";

                    result += "Required P for next DF: ";
                    for (int j = 0; j < testResult.Variants - 1; j++)
                    {
                        result += "[" + j.ToString() + "]: " + testResult.ReqPVal[j] + "\t";
                    }
                    result += "\n";

                    result += "Is significant: ";
                    for (int j = 0; j < testResult.Variants - 1; j++)
                    {
                        result += "[" + j.ToString() + "]: " + testResult.ReachedSignificance[j] + "\t";
                    }
                    result += "\n";


                    result += "\nNote: [0] is the control variant";
                }
                else
                {
                    result += "Results not analysed yet.\n";
                }

                result += "\n----\n";
            }

            result += "\n\n********** MVT STATS **********\n\n";
            _CLI.Out(result, false, false);
        }
Example #5
0
        public static string TestHistoryAsCSV()
        {
            string result = "";

            MultiVariateTest[] allTests = MVTHistory.GetAllTests();

            if (allTests != null && allTests.Length > 0)
            {
                result =
                    "Test ID," + "Status," + "Launched," + "Finished," + "Type," + "Control variant," + "Test variant,"
                    + "CTR Visitors," + "TST Visitors," + "CTR Converted," + "TST Converted," + "CTR Conversion %," + "TST Conversion %," + "CTR Std Dev," + "TST Std Dev,"
                    + "T-Score," + "Degree of Freedom," + "Next Degree of Freedom," + "Required P for next DFree"
                    + "\n";

                for (int i = 0; i < allTests.Length; i++)
                {
                    MVTResult TResult = allTests[i].GetResult;

                    //Test ID
                    result += allTests[i].Id.ToString() + ",";

                    //Status
                    switch (allTests[i].GetResultType())
                    {
                    case TestResultType.Inconclusive:
                        result += "Inconclusive,";
                        break;

                    case TestResultType.Live:
                        result += "Live,";
                        break;

                    case TestResultType.Negative:
                        result += "Negative,";
                        break;

                    case TestResultType.Positive:
                        result += "Positive,";
                        break;

                    case TestResultType.None:
                        result += "None,";
                        break;
                    }

                    //Launched
                    DateTime?launched = allTests[i].GetLaunchDateTime;
                    if (launched != null)
                    {
                        result += launched.ToString() + ",";
                    }
                    else
                    {
                        result += ",";
                    }

                    //Finished
                    DateTime?finished = allTests[i].GetCompletionDate;
                    if (finished != null)
                    {
                        result += finished.ToString() + ",";
                    }
                    else
                    {
                        result += ",";
                    }

                    //Type
                    if (allTests[i].TestType == TestTypesEnum.ADDTOCARTBUTTON)
                    {
                        result += "Add to Cart Btn,";
                    }
                    else
                    {
                        result += ",";
                    }

                    //Control variant description
                    AddToCartButtonVariation ctrVar = (AddToCartButtonVariation)allTests[i].GetVariant(0).GetVariationType();
                    if (ctrVar.colorPalette == null && ctrVar.addToCartTextCopies == null)
                    {
                        result += "No changes";
                    }
                    else
                    {
                        if (ctrVar.colorPalette != null)
                        {
                            result += "Colour: " + ctrVar.colorPalette + "; ";
                        }

                        if (ctrVar.addToCartTextCopies != null)
                        {
                            result += "Copy change: " + ctrVar.addToCartTextCopies;
                        }
                    }
                    result += ",";

                    //Test variant description
                    AddToCartButtonVariation tstVar = (AddToCartButtonVariation)allTests[i].GetVariant(1).GetVariationType();
                    if (tstVar.colorPalette == null && tstVar.addToCartTextCopies == null)
                    {
                        result += "No changes";
                    }
                    else
                    {
                        if (tstVar.colorPalette != null)
                        {
                            result += "Colour: " + tstVar.colorPalette + "; ";
                        }

                        if (tstVar.addToCartTextCopies != null)
                        {
                            result += "Copy change: " + tstVar.addToCartTextCopies;
                        }
                    }
                    result += ",";



                    if (TResult != null)
                    {
                        if (TResult.SampleSize.Length > 1 &&
                            TResult.Means.Length > 1 &&
                            TResult.ConvertedSize.Length > 1 &&
                            TResult.STDevs.Length > 1)
                        {
                            //CTR visitors
                            result += TResult.SampleSize[0].ToString() + ",";

                            //TST visitors
                            result += TResult.SampleSize[1].ToString() + ",";

                            //CTR converted
                            result += TResult.ConvertedSize[0].ToString() + ",";

                            //TST converted
                            result += TResult.ConvertedSize[1].ToString() + ",";

                            //CTR conversion
                            result += TResult.Means[0].ToString() + ",";

                            //TST conversion
                            result += TResult.Means[1].ToString() + ",";

                            //CTR std dev
                            result += TResult.STDevs[0].ToString() + ",";

                            //TST std dev
                            result += TResult.STDevs[1].ToString() + ",";

                            //T-score
                            result += TResult.TScores[0].ToString() + ",";

                            //Degree of freedom
                            result += TResult.DFree[0].ToString() + ",";

                            //Next degree of freedom
                            result += TResult.NextDFree[0].ToString() + ",";

                            //Required DFree
                            result += TResult.ReqPVal[0].ToString();
                        }
                    }

                    result += "\n";
                }
            }
            else
            {
                result = "No tests launched yet.";
            }

            return(result);
        }