Example #1
0
        // Performs evaluation with the truncation level set up to 10 search results within a query.
        // This is a temporary workaround for this issue: https://github.com/dotnet/machinelearning/issues/2728.
        public static void EvaluateMetrics(MLContext mlContext, IDataView predictions, int truncationLevel)
        {
            if (truncationLevel < 1 || truncationLevel > 10)
            {
                throw new InvalidOperationException("Currently metrics are only supported for 1 to 10 truncation levels.");
            }

            //  Uses reflection to set the truncation level before calling evaluate.
            var mlAssembly   = typeof(TextLoader).Assembly;
            var rankEvalType = mlAssembly.DefinedTypes.Where(t => t.Name.Contains("RankingEvaluator")).First();

            var evalArgsType = rankEvalType.GetNestedType("Arguments");
            var evalArgs     = Activator.CreateInstance(rankEvalType.GetNestedType("Arguments"));

            var dcgLevel = evalArgsType.GetField("DcgTruncationLevel");

            dcgLevel.SetValue(evalArgs, truncationLevel);

            var ctor      = rankEvalType.GetConstructors().First();
            var evaluator = ctor.Invoke(new object[] { mlContext, evalArgs });

            var            evaluateMethod = rankEvalType.GetMethod("Evaluate");
            RankingMetrics metrics        = (RankingMetrics)evaluateMethod.Invoke(evaluator, new object[] { predictions, "Label", "GroupId", "Score" });

            Console.WriteLine($"DCG: {string.Join(", ", metrics.DiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F4}").ToArray())}");

            Console.WriteLine($"NDCG: {string.Join(", ", metrics.NormalizedDiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F4}").ToArray())}\n");
        }
        private static RankingMetrics RankingDelta(
            RankingMetrics a, RankingMetrics b)
        {
            var dcg  = ComputeArrayDeltas(a.Dcg, b.Dcg);
            var ndcg = ComputeArrayDeltas(a.Ndcg, b.Ndcg);

            return(new RankingMetrics(dcg: dcg, ndcg: ndcg));
        }
Example #3
0
 public static void PrintRankingMetrics(string name, RankingMetrics metrics, uint optimizationMetricTruncationLevel)
 {
     Console.WriteLine($"************************************************************");
     Console.WriteLine($"*             Metrics for {name} ranking model              ");
     Console.WriteLine($"*-----------------------------------------------------------");
     Console.WriteLine($"    Normalized Discounted Cumulative Gain (NDCG@{optimizationMetricTruncationLevel}) = {metrics?.NormalizedDiscountedCumulativeGains?[(int)optimizationMetricTruncationLevel - 1] ?? double.NaN:0.####}, a value from 0 and 1, where closer to 1.0 is better");
     Console.WriteLine($"    Discounted Cumulative Gain (DCG@{optimizationMetricTruncationLevel}) = {metrics?.DiscountedCumulativeGains?[(int)optimizationMetricTruncationLevel - 1] ?? double.NaN:0.####}");
 }
        public static void EvaluateMetrics(MLContext mlContext, IDataView predictions)
        {
            RankingMetrics metrics = mlContext.Ranking.Evaluate(predictions);

            Console.WriteLine($"DCG: {string.Join(", ", metrics.DiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F4}").ToArray())}");

            Console.WriteLine($"NDCG: {string.Join(", ", metrics.NormalizedDiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F4}").ToArray())}\n");
        }
Example #5
0
        // To evaluate the accuracy of the model's predicted rankings, prints out the Discounted Cumulative Gain and Normalized Discounted Cumulative Gain for search queries.
        public static void EvaluateMetrics(MLContext mlContext, IDataView predictions)
        {
            // Evaluate the metrics for the data using NDCG; by default, metrics for the up to 3 search results in the query are reported (e.g. NDCG@3).
            RankingMetrics metrics = mlContext.Ranking.Evaluate(predictions);

            Console.WriteLine($"DCG: {string.Join(", ", metrics.DiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F4}").ToArray())}");

            Console.WriteLine($"NDCG: {string.Join(", ", metrics.NormalizedDiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F4}").ToArray())}\n");
        }
 // Pretty-print RankerMetrics objects.
 public static void PrintMetrics(RankingMetrics metrics)
 {
     Console.WriteLine("DCG: " + string.Join(", ",
                                             metrics.DiscountedCumulativeGains.Select(
                                                 (d, i) => (i + 1) + ":" + d + ":F2").ToArray()));
     Console.WriteLine("NDCG: " + string.Join(", ",
                                              metrics.NormalizedDiscountedCumulativeGains.Select(
                                                  (d, i) => (i + 1) + ":" + d + ":F2").ToArray()));
 }
Example #7
0
 /// <summary>
 /// Check that a <see cref="RankingMetrics"/> object is valid.
 /// </summary>
 /// <param name="metrics">The metrics object.</param>
 public static void AssertMetrics(RankingMetrics metrics)
 {
     foreach (var dcg in metrics.Dcg)
     {
         Assert.True(dcg >= 0);
     }
     foreach (var ndcg in metrics.Ndcg)
     {
         Assert.InRange(ndcg, 0, 100);
     }
 }
 /// <summary>
 /// Check that a <see cref="RankingMetrics"/> object is valid.
 /// </summary>
 /// <param name="metrics">The metrics object.</param>
 public static void AssertMetrics(RankingMetrics metrics)
 {
     foreach (var dcg in metrics.DiscountedCumulativeGains)
     {
         Assert.True(dcg >= 0);
     }
     foreach (var ndcg in metrics.NormalizedDiscountedCumulativeGains)
     {
         Assert.InRange(ndcg, 0, 100);
     }
 }
Example #9
0
 private static double GetScore(RankingMetrics metrics, RankingMetric metric)
 {
     return(new RankingMetricsAgent(null, metric).GetScore(metrics));
 }
Example #10
0
 private static void PrintMetrics(RankingMetrics metrics)
 {
     Console.WriteLine($"NormalizedDiscountedCumulativeGains: {metrics.NormalizedDiscountedCumulativeGains}");
     Console.WriteLine($"DiscountedCumulativeGains: {metrics.DiscountedCumulativeGains}");
 }
Example #11
0
        public static void Run()
        {
            MLContext mlContext = new MLContext();

            // STEP 1: Load data
            IDataView trainDataView = mlContext.Data.LoadFromTextFile <SearchData>(TrainDataPath, hasHeader: true, separatorChar: ',');
            IDataView testDataView  = mlContext.Data.LoadFromTextFile <SearchData>(TestDataPath, hasHeader: true, separatorChar: ',');

            // STEP 2: Run AutoML experiment
            Console.WriteLine($"Running AutoML recommendation experiment for {ExperimentTime} seconds...");
            ExperimentResult <RankingMetrics> experimentResult = mlContext.Auto()
                                                                 .CreateRankingExperiment(new RankingExperimentSettings()
            {
                MaxExperimentTimeInSeconds = ExperimentTime
            })
                                                                 .Execute(trainDataView, testDataView,
                                                                          new ColumnInformation()
            {
                LabelColumnName   = LabelColumnName,
                GroupIdColumnName = GroupColumnName
            });

            // STEP 3: Print metric from best model
            RunDetail <RankingMetrics> bestRun = experimentResult.BestRun;

            Console.WriteLine($"Total models produced: {experimentResult.RunDetails.Count()}");
            Console.WriteLine($"Best model's trainer: {bestRun.TrainerName}");
            Console.WriteLine($"Metrics of best model from validation data --");
            PrintMetrics(bestRun.ValidationMetrics);

            // STEP 5: Evaluate test data
            IDataView      testDataViewWithBestScore = bestRun.Model.Transform(testDataView);
            RankingMetrics testMetrics = mlContext.Ranking.Evaluate(testDataViewWithBestScore, labelColumnName: LabelColumnName);

            Console.WriteLine($"Metrics of best model on test data --");
            PrintMetrics(testMetrics);

            // STEP 6: Save the best model for later deployment and inferencing
            mlContext.Model.Save(bestRun.Model, trainDataView.Schema, ModelPath);

            // STEP 7: Create prediction engine from the best trained model
            var predictionEngine = mlContext.Model.CreatePredictionEngine <SearchData, SearchDataPrediction>(bestRun.Model);

            // STEP 8: Initialize a new test, and get the prediction
            var testPage = new SearchData
            {
                GroupId  = "1",
                Features = 9,
                Label    = 1
            };
            var prediction = predictionEngine.Predict(testPage);

            Console.WriteLine($"Predicted rating for: {prediction.Prediction}");

            // New Page
            testPage = new SearchData
            {
                GroupId  = "2",
                Features = 2,
                Label    = 9
            };
            prediction = predictionEngine.Predict(testPage);
            Console.WriteLine($"Predicted: {prediction.Prediction}");

            Console.WriteLine("Press any key to continue...");
            Console.ReadKey();
        }
Example #12
0
        private static bool IsPerfectModel(RankingMetrics metrics, RankingMetric metric, uint dcgTruncationLevel)
        {
            var metricsAgent = new RankingMetricsAgent(null, metric, dcgTruncationLevel);

            return(IsPerfectModel(metricsAgent, metrics));
        }
 private static double GetScore(RankingMetrics metrics, RankingMetric metric, string groupIdColumnName)
 {
     return(new RankingMetricsAgent(null, metric, groupIdColumnName).GetScore(metrics));
 }
Example #14
0
 private static double GetScore(RankingMetrics metrics, RankingMetric metric, uint dcgTruncationLevel)
 {
     return(new RankingMetricsAgent(null, metric, dcgTruncationLevel).GetScore(metrics));
 }
        private static TMetrics GetAverageMetrics(IEnumerable <TMetrics> metrics, TMetrics metricsClosestToAvg)
        {
            if (typeof(TMetrics) == typeof(BinaryClassificationMetrics))
            {
                var newMetrics = metrics.Select(x => x as BinaryClassificationMetrics);
                Contracts.Assert(newMetrics != null);

                var result = new BinaryClassificationMetrics(
                    auc: GetAverageOfNonNaNScores(newMetrics.Select(x => x.AreaUnderRocCurve)),
                    accuracy: GetAverageOfNonNaNScores(newMetrics.Select(x => x.Accuracy)),
                    positivePrecision: GetAverageOfNonNaNScores(newMetrics.Select(x => x.PositivePrecision)),
                    positiveRecall: GetAverageOfNonNaNScores(newMetrics.Select(x => x.PositiveRecall)),
                    negativePrecision: GetAverageOfNonNaNScores(newMetrics.Select(x => x.NegativePrecision)),
                    negativeRecall: GetAverageOfNonNaNScores(newMetrics.Select(x => x.NegativeRecall)),
                    f1Score: GetAverageOfNonNaNScores(newMetrics.Select(x => x.F1Score)),
                    auprc: GetAverageOfNonNaNScores(newMetrics.Select(x => x.AreaUnderPrecisionRecallCurve)),
                    // Return ConfusionMatrix from the fold closest to average score
                    confusionMatrix: (metricsClosestToAvg as BinaryClassificationMetrics).ConfusionMatrix);
                return(result as TMetrics);
            }

            if (typeof(TMetrics) == typeof(MulticlassClassificationMetrics))
            {
                var newMetrics = metrics.Select(x => x as MulticlassClassificationMetrics);
                Contracts.Assert(newMetrics != null);

                var result = new MulticlassClassificationMetrics(
                    accuracyMicro: GetAverageOfNonNaNScores(newMetrics.Select(x => x.MicroAccuracy)),
                    accuracyMacro: GetAverageOfNonNaNScores(newMetrics.Select(x => x.MacroAccuracy)),
                    logLoss: GetAverageOfNonNaNScores(newMetrics.Select(x => x.LogLoss)),
                    logLossReduction: GetAverageOfNonNaNScores(newMetrics.Select(x => x.LogLossReduction)),
                    topKPredictionCount: newMetrics.ElementAt(0).TopKPredictionCount,
                    topKAccuracy: GetAverageOfNonNaNScores(newMetrics.Select(x => x.TopKAccuracy)),
                    // Return PerClassLogLoss and ConfusionMatrix from the fold closest to average score
                    perClassLogLoss: (metricsClosestToAvg as MulticlassClassificationMetrics).PerClassLogLoss.ToArray(),
                    confusionMatrix: (metricsClosestToAvg as MulticlassClassificationMetrics).ConfusionMatrix);
                return(result as TMetrics);
            }

            if (typeof(TMetrics) == typeof(RegressionMetrics))
            {
                var newMetrics = metrics.Select(x => x as RegressionMetrics);
                Contracts.Assert(newMetrics != null);

                var result = new RegressionMetrics(
                    l1: GetAverageOfNonNaNScores(newMetrics.Select(x => x.MeanAbsoluteError)),
                    l2: GetAverageOfNonNaNScores(newMetrics.Select(x => x.MeanSquaredError)),
                    rms: GetAverageOfNonNaNScores(newMetrics.Select(x => x.RootMeanSquaredError)),
                    lossFunction: GetAverageOfNonNaNScores(newMetrics.Select(x => x.LossFunction)),
                    rSquared: GetAverageOfNonNaNScores(newMetrics.Select(x => x.RSquared)));
                return(result as TMetrics);
            }

            if (typeof(TMetrics) == typeof(RankingMetrics))
            {
                var newMetrics = metrics.Select(x => x as RankingMetrics);
                Contracts.Assert(newMetrics != null);

                var result = new RankingMetrics(
                    dcg: GetAverageOfNonNaNScoresInNestedEnumerable(newMetrics.Select(x => x.DiscountedCumulativeGains)),
                    ndcg: GetAverageOfNonNaNScoresInNestedEnumerable(newMetrics.Select(x => x.NormalizedDiscountedCumulativeGains)));
                return(result as TMetrics);
            }

            throw new NotImplementedException($"Metric {typeof(TMetrics)} not implemented");
        }
Example #16
0
        private static bool IsPerfectModel(RankingMetrics metrics, RankingMetric metric)
        {
            var metricsAgent = new RankingMetricsAgent(null, metric);

            return(IsPerfectModel(metricsAgent, metrics));
        }
Example #17
0
 internal static void PrintIterationMetrics(int iteration, string trainerName, RankingMetrics metrics, double?runtimeInSeconds)
 {
     CreateRow($"{iteration,-4} {trainerName,-15} {metrics?.NormalizedDiscountedCumulativeGains[0] ?? double.NaN,9:F4} {metrics?.NormalizedDiscountedCumulativeGains[2] ?? double.NaN,9:F4} {metrics?.NormalizedDiscountedCumulativeGains[9] ?? double.NaN,9:F4} {metrics?.DiscountedCumulativeGains[9] ?? double.NaN,9:F4} {runtimeInSeconds.Value,9:F1}", Width);
 }
        private static bool IsPerfectModel(RankingMetrics metrics, RankingMetric metric, string groupIdColumnName)
        {
            var metricsAgent = new RankingMetricsAgent(null, metric, groupIdColumnName);

            return(IsPerfectModel(metricsAgent, metrics));
        }