/// <summary>Evaluate on the folds of a dataset split</summary> /// <param name="recommender">a rating predictor</param> /// <param name="split">a rating dataset split</param> /// <param name="compute_fit">if set to true measure fit on the training data as well</param> /// <param name="show_fold_results">set to true to print per-fold results to STDERR</param> /// <returns>a dictionary containing the average results over the different folds of the split</returns> public static RatingPredictionEvaluationResults DoCrossValidation( this RatingPredictor recommender, ISplit<IRatings> split, bool compute_fit = false, bool show_fold_results = false) { var fold_results = new RatingPredictionEvaluationResults[split.NumberOfFolds]; Parallel.For(0, (int) split.NumberOfFolds, i => { try { var split_recommender = (RatingPredictor) recommender.Clone(); // to avoid changes in recommender split_recommender.Ratings = split.Train[i]; if (recommender is ITransductiveRatingPredictor) ((ITransductiveRatingPredictor) split_recommender).AdditionalFeedback = split.Test[i]; split_recommender.Train(); fold_results[i] = Ratings.Evaluate(split_recommender, split.Test[i]); if (compute_fit) fold_results[i]["fit"] = (float) split_recommender.ComputeFit(); if (show_fold_results) Console.Error.WriteLine("fold {0} {1}", i, fold_results[i]); } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw e; } }); return new RatingPredictionEvaluationResults(fold_results); }
/// <summary>Find the the parameters resulting in the minimal results for a given evaluation measure (2D)</summary> /// <remarks>The recommender will be set to the best parameter value after calling this method.</remarks> /// <param name="evaluation_measure">the name of the evaluation measure</param> /// <param name="hp_name1">the name of the first hyperparameter to optimize</param> /// <param name="hp_values1">the values of the first hyperparameter to try out</param> /// <param name="hp_name2">the name of the second hyperparameter to optimize</param> /// <param name="hp_values2">the values of the second hyperparameter to try out</param> /// <param name="recommender">the recommender</param> /// <param name="split">the dataset split to use</param> /// <returns>the best (lowest) average value for the hyperparameter</returns> public static double FindMinimum( string evaluation_measure, string hp_name1, string hp_name2, double[] hp_values1, double[] hp_values2, RatingPredictor recommender, ISplit <IRatings> split) { double min_result = double.MaxValue; int min_i = -1; int min_j = -1; for (int i = 0; i < hp_values1.Length; i++) { for (int j = 0; j < hp_values2.Length; j++) { recommender.SetProperty(hp_name1, hp_values1[i].ToString(CultureInfo.InvariantCulture)); recommender.SetProperty(hp_name2, hp_values2[j].ToString(CultureInfo.InvariantCulture)); Console.Error.WriteLine("reg_u={0} reg_i={1}", hp_values1[i].ToString(CultureInfo.InvariantCulture), hp_values2[j].ToString(CultureInfo.InvariantCulture)); // TODO this is not generic double result = recommender.DoCrossValidation(split)[evaluation_measure]; if (result < min_result) { min_i = i; min_j = j; min_result = result; } } } // set to best hyperparameter values recommender.SetProperty(hp_name1, hp_values1[min_i].ToString(CultureInfo.InvariantCulture)); recommender.SetProperty(hp_name2, hp_values2[min_j].ToString(CultureInfo.InvariantCulture)); return(min_result); }
/// <summary>Find the the parameters resulting in the minimal results for a given evaluation measure (1D)</summary> /// <remarks>The recommender will be set to the best parameter value after calling this method.</remarks> /// <param name="evaluation_measure">the name of the evaluation measure</param> /// <param name="hyperparameter_name">the name of the hyperparameter to optimize</param> /// <param name="hyperparameter_values">the values of the hyperparameter to try out</param> /// <param name="recommender">the recommender</param> /// <param name="split">the dataset split to use</param> /// <returns>the best (lowest) average value for the hyperparameter</returns> public static double FindMinimum( string evaluation_measure, string hyperparameter_name, double[] hyperparameter_values, RatingPredictor recommender, ISplit<IRatings> split) { double min_result = double.MaxValue; int min_i = -1; for (int i = 0; i < hyperparameter_values.Length; i++) { recommender.SetProperty(hyperparameter_name, hyperparameter_values[i].ToString(CultureInfo.InvariantCulture)); double result = recommender.DoCrossValidation(split)[evaluation_measure]; if (result < min_result) { min_i = i; min_result = result; } } recommender.SetProperty(hyperparameter_name, hyperparameter_values[min_i].ToString(CultureInfo.InvariantCulture)); return min_result; }
public Cube(byte[] redArray, byte[] greenArray, byte[] blueArray, ISelect select, ISplit split) { ColorCount = redArray.Length; Initialize(redArray, greenArray, blueArray); SelectType = select; SplitType = split; }
/// <summary>Find the the parameters resulting in the minimal results for a given evaluation measure (2D)</summary> /// <remarks>The recommender will be set to the best parameter value after calling this method.</remarks> /// <param name="evaluation_measure">the name of the evaluation measure</param> /// <param name="hp_name1">the name of the first hyperparameter to optimize</param> /// <param name="hp_values1">the values of the first hyperparameter to try out</param> /// <param name="hp_name2">the name of the second hyperparameter to optimize</param> /// <param name="hp_values2">the values of the second hyperparameter to try out</param> /// <param name="recommender">the recommender</param> /// <param name="split">the dataset split to use</param> /// <returns>the best (lowest) average value for the hyperparameter</returns> public static double FindMinimum( string evaluation_measure, string hp_name1, string hp_name2, double[] hp_values1, double[] hp_values2, RatingPredictor recommender, ISplit<IRatings> split) { double min_result = double.MaxValue; int min_i = -1; int min_j = -1; for (int i = 0; i < hp_values1.Length; i++) for (int j = 0; j < hp_values2.Length; j++) { recommender.SetProperty(hp_name1, hp_values1[i].ToString(CultureInfo.InvariantCulture)); recommender.SetProperty(hp_name2, hp_values2[j].ToString(CultureInfo.InvariantCulture)); Console.Error.WriteLine("reg_u={0} reg_i={1}", hp_values1[i].ToString(CultureInfo.InvariantCulture), hp_values2[j].ToString(CultureInfo.InvariantCulture)); // TODO this is not generic double result = recommender.DoCrossValidation(split)[evaluation_measure]; if (result < min_result) { min_i = i; min_j = j; min_result = result; } } // set to best hyperparameter values recommender.SetProperty(hp_name1, hp_values1[min_i].ToString(CultureInfo.InvariantCulture)); recommender.SetProperty(hp_name2, hp_values2[min_j].ToString(CultureInfo.InvariantCulture)); return min_result; }
/// <summary>Find the the parameters resulting in the minimal results for a given evaluation measure (1D)</summary> /// <remarks>The recommender will be set to the best parameter value after calling this method.</remarks> /// <param name="evaluation_measure">the name of the evaluation measure</param> /// <param name="hyperparameter_name">the name of the hyperparameter to optimize</param> /// <param name="hyperparameter_values">the values of the hyperparameter to try out</param> /// <param name="recommender">the recommender</param> /// <param name="split">the dataset split to use</param> /// <returns>the best (lowest) average value for the hyperparameter</returns> public static double FindMinimum( string evaluation_measure, string hyperparameter_name, double[] hyperparameter_values, RatingPredictor recommender, ISplit <IRatings> split) { double min_result = double.MaxValue; int min_i = -1; for (int i = 0; i < hyperparameter_values.Length; i++) { recommender.SetProperty(hyperparameter_name, hyperparameter_values[i].ToString(CultureInfo.InvariantCulture)); double result = recommender.DoCrossValidation(split)[evaluation_measure]; if (result < min_result) { min_i = i; min_result = result; } } recommender.SetProperty(hyperparameter_name, hyperparameter_values[min_i].ToString(CultureInfo.InvariantCulture)); return(min_result); }
private void Init() { this.split = new RatingsSimpleSplit(recommender.Ratings, split_ratio); //this.split = new RatingCrossValidationSplit(recommender.Ratings, 5); // TODO manage this via reflection? if (recommender is UserItemBaseline) { this.hp_names = new string[] { "reg_u", "reg_i" }; this.initial_hp_values = CreateInitialValues( new double[][] { new double[] { 25, 10 }, new double[] { 10, 25 }, new double[] { 2, 5 }, new double[] { 5, 2 }, new double[] { 1, 4 }, new double[] { 4, 1 }, new double[] { 3, 3 }, } ); } else if (recommender is BiasedMatrixFactorization) { this.hp_names = new string[] { "regularization", "bias_reg" }; this.initial_hp_values = CreateInitialValues( // TODO reg_u and reg_i (in a second step?) new double[][] { new double[] { 0.1, 0 }, new double[] { 0.01, 0 }, new double[] { 0.0001, 0 }, new double[] { 0.00001, 0 }, new double[] { 0.1, 0.0001 }, new double[] { 0.01, 0.0001 }, new double[] { 0.0001, 0.0001 }, new double[] { 0.00001, 0.0001 }, } ); } else if (recommender is MatrixFactorization) { this.hp_names = new string[] { "regularization" }; // TODO normal interval search could be more efficient this.initial_hp_values = CreateInitialValues( new double[][] { new double[] { 0.1 }, new double[] { 0.01 }, new double[] { 0.0001 }, new double[] { 0.00001 }, } ); } // TODO kNN-based methods else { throw new Exception("not prepared for type " + recommender.GetType().ToString()); } }
static double Run(RatingPredictor recommender, ISplit<IRatings> split, string hp_string, string evaluation_measure) { recommender.Configure(hp_string); double result = recommender.DoCrossValidation(split)[evaluation_measure]; Console.Error.WriteLine("Nelder-Mead: {0}: {1}", hp_string, result.ToString(CultureInfo.InvariantCulture)); return result; }
static double Run(RatingPredictor recommender, ISplit <IRatings> split, string hp_string, string evaluation_measure) { Recommender.Configure(recommender, hp_string); double result = Eval.Ratings.EvaluateOnSplit(recommender, split)[evaluation_measure]; Console.Error.WriteLine("Nelder-Mead: {0}: {1}", hp_string, result.ToString(CultureInfo.InvariantCulture)); return(result); }
/// <summary>Evaluate on the folds of a dataset split</summary> /// <param name="recommender">an item recommender</param> /// <param name="split">a dataset split</param> /// <param name="test_users">a collection of integers with all test users</param> /// <param name="candidate_items">a collection of integers with all candidate items</param> /// <param name="candidate_item_mode">the mode used to determine the candidate items</param> /// <param name="compute_fit">if set to true measure fit on the training data as well</param> /// <param name="show_results">set to true to print results to STDERR</param> /// <returns>a dictionary containing the average results over the different folds of the split</returns> public static ItemRecommendationEvaluationResults DoCrossValidation( this IRecommender recommender, ISplit<IPosOnlyFeedback> split, IList<int> test_users, IList<int> candidate_items, CandidateItems candidate_item_mode = CandidateItems.OVERLAP, bool compute_fit = false, bool show_results = false) { var avg_results = new ItemRecommendationEvaluationResults(); if (!(recommender is ItemRecommender)) throw new ArgumentException("recommender must be of type ItemRecommender"); Parallel.For(0, (int) split.NumberOfFolds, fold => { try { var split_recommender = (ItemRecommender) recommender.Clone(); // avoid changes in recommender split_recommender.Feedback = split.Train[fold]; split_recommender.Train(); var fold_results = Items.Evaluate(split_recommender, split.Test[fold], split.Train[fold], test_users, candidate_items, candidate_item_mode); if (compute_fit) fold_results["fit"] = (float) split_recommender.ComputeFit(); // thread-safe stats lock (avg_results) foreach (var key in fold_results.Keys) if (avg_results.ContainsKey(key)) avg_results[key] += fold_results[key]; else avg_results[key] = fold_results[key]; if (show_results) Console.Error.WriteLine("fold {0} {1}", fold, fold_results); } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); foreach (var key in Items.Measures) avg_results[key] /= split.NumberOfFolds; avg_results["num_users"] /= split.NumberOfFolds; avg_results["num_items"] /= split.NumberOfFolds; if (compute_fit) avg_results["fit"] /= split.NumberOfFolds; return avg_results; }
/// <summary>Evaluate on the folds of a dataset split</summary> /// <param name="recommender">an item recommender</param> /// <param name="split">a dataset split</param> /// <param name="candidate_items">a collection of integers with all candidate items</param> /// <param name="candidate_item_mode">the mode used to determine the candidate items</param> /// <param name="compute_fit">if set to true measure fit on the training data as well</param> /// <param name="show_results">set to true to print results to STDERR</param> /// <returns>a dictionary containing the average results over the different folds of the split</returns> static public EvaluationResults DoRatingBasedRankingCrossValidation( this RatingPredictor recommender, ISplit<IRatings> split, IList<int> candidate_items, CandidateItems candidate_item_mode = CandidateItems.OVERLAP, bool compute_fit = false, bool show_results = false) { var avg_results = new ItemRecommendationEvaluationResults(); Parallel.For(0, (int) split.NumberOfFolds, fold => { try { var split_recommender = (RatingPredictor) recommender.Clone(); // avoid changes in recommender split_recommender.Ratings = split.Train[fold]; split_recommender.Train(); var test_data_posonly = new PosOnlyFeedback<SparseBooleanMatrix>(split.Test[fold]); var training_data_posonly = new PosOnlyFeedback<SparseBooleanMatrix>(split.Train[fold]); IList<int> test_users = test_data_posonly.AllUsers; var fold_results = Items.Evaluate(split_recommender, test_data_posonly, training_data_posonly, test_users, candidate_items, candidate_item_mode); if (compute_fit) fold_results["fit"] = (float) split_recommender.ComputeFit(); // thread-safe stats lock (avg_results) foreach (var key in fold_results.Keys) if (avg_results.ContainsKey(key)) avg_results[key] += fold_results[key]; else avg_results[key] = fold_results[key]; if (show_results) Console.Error.WriteLine("fold {0} {1}", fold, fold_results); } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); foreach (var key in Items.Measures) avg_results[key] /= split.NumberOfFolds; avg_results["num_users"] /= split.NumberOfFolds; avg_results["num_items"] /= split.NumberOfFolds; return avg_results; }
/// <summary>Find the the parameters resulting in the minimal results for a given evaluation measure (1D)</summary> /// <remarks>The recommender will be set to the best parameter value after calling this method.</remarks> /// <param name="evaluation_measure">the name of the evaluation measure</param> /// <param name="hp_name">the name of the hyperparameter to optimize</param> /// <param name="hp_values">the logarithms of the values of the hyperparameter to try out</param> /// <param name="basis">the basis to use for the logarithms</param> /// <param name="recommender">the recommender</param> /// <param name="split">the dataset split to use</param> /// <returns>the best (lowest) average value for the hyperparameter</returns> public static double FindMinimumExponential(string evaluation_measure, string hp_name, double[] hp_values, double basis, RatingPrediction.RatingPredictor recommender, ISplit <IRatings> split) { var new_hp_values = new double[hp_values.Length]; for (int i = 0; i < hp_values.Length; i++) { new_hp_values[i] = Math.Pow(basis, hp_values[i]); } return(FindMinimum(evaluation_measure, hp_name, new_hp_values, recommender, split)); }
public Cube(byte[] pixels, ISelect select, ISplit split) { ColorCount = pixels.Length / 4; SelectType = select; SplitType = split; var redArray = new byte[ColorCount]; var greenArray = new byte[ColorCount]; var blueArray = new byte[ColorCount]; for (int i = 0; i < ColorCount; i++) { redArray[i] = pixels[i * 4 + 2]; greenArray[i] = pixels[i * 4 + 1]; blueArray[i] = pixels[i * 4]; } Initialize(redArray, greenArray, blueArray); }
/// <summary>Evaluate on the folds of a dataset split</summary> /// <param name="recommender">a rating predictor</param> /// <param name="split">a rating dataset split</param> /// <param name="compute_fit">if set to true measure fit on the training data as well</param> /// <param name="show_results">set to true to print results to STDERR</param> /// <returns>a dictionary containing the average results over the different folds of the split</returns> public static RatingPredictionEvaluationResults DoCrossValidation( this RatingPredictor recommender, ISplit<IRatings> split, bool compute_fit = false, bool show_results = false) { var avg_results = new RatingPredictionEvaluationResults(); Parallel.For(0, (int) split.NumberOfFolds, i => { try { var split_recommender = (RatingPredictor) recommender.Clone(); // to avoid changes in recommender split_recommender.Ratings = split.Train[i]; if (recommender is ITransductiveRatingPredictor) ((ITransductiveRatingPredictor) split_recommender).AdditionalFeedback = split.Test[i]; split_recommender.Train(); var fold_results = Ratings.Evaluate(split_recommender, split.Test[i]); if (compute_fit) fold_results["fit"] = (float) split_recommender.ComputeFit(); // thread-safe stats lock (avg_results) foreach (var key in fold_results.Keys) if (avg_results.ContainsKey(key)) avg_results[key] += fold_results[key]; else avg_results[key] = fold_results[key]; if (show_results) Console.Error.WriteLine("fold {0} {1}", i, fold_results); } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw e; } }); foreach (var key in Ratings.Measures) avg_results[key] /= split.NumberOfFolds; return avg_results; }
/// <summary>Evaluate on the folds of a dataset split</summary> /// <param name="recommender">a rating predictor</param> /// <param name="split">a rating dataset split</param> /// <param name="compute_fit">if set to true measure fit on the training data as well</param> /// <param name="show_fold_results">set to true to print per-fold results to STDERR</param> /// <returns>a dictionary containing the average results over the different folds of the split</returns> static public RatingPredictionEvaluationResults DoCrossValidation( this RatingPredictor recommender, ISplit <IRatings> split, bool compute_fit = false, bool show_fold_results = false) { var fold_results = new RatingPredictionEvaluationResults[split.NumberOfFolds]; Parallel.For(0, (int)split.NumberOfFolds, i => { try { var split_recommender = (RatingPredictor)recommender.Clone(); // to avoid changes in recommender split_recommender.Ratings = split.Train[i]; if (recommender is ITransductiveRatingPredictor) { ((ITransductiveRatingPredictor)split_recommender).AdditionalFeedback = split.Test[i]; } split_recommender.Train(); fold_results[i] = Ratings.Evaluate(split_recommender, split.Test[i]); if (compute_fit) { fold_results[i]["fit"] = (float)split_recommender.ComputeFit(); } if (show_fold_results) { Console.Error.WriteLine("fold {0} {1}", i, fold_results[i]); } } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); return(new RatingPredictionEvaluationResults(fold_results)); }
public void Init() { var excelSplit = new ExcelSplit("ExcelTutorialR1 — edytowalny"); ExcSampleSplit = excelSplit; SplitXml = excelSplit; ExcSampleMerge = excelSplit; DocValidator = new OpenXmlValidator(); var parts = PartsSelectionTreeElementMock.GetListMock(); CreateSplitXmlBinary = SplitXml.CreateSplitXml(parts); ExcSampleDocInMemory = new MemoryStream(File.ReadAllBytes(@"../../../Files/ExcelTutorialR1 — edytowalny.xlsx")); byte[] sampleXmlBinary = File.ReadAllBytes(@"../../../Files/split_ExcelTutorialR1 — edytowalny.xlsx_20170304230639351.xml"); excelSplit.OpenAndSearchDocument(ExcSampleDocInMemory, new MemoryStream(sampleXmlBinary)); MergeXmlBinary = ExcSampleMerge.CreateMergeXml(); ErrorsCount = 0; WarningsCount = 0; }
public void Init() { var wordSplit = new WordSplit("demo"); WordSampleMerge = wordSplit; WordSampleSplit = wordSplit; SplitXml = wordSplit; DocValidator = new OpenXmlValidator(); var parts = PartsSelectionTreeElementMock.GetListMock(); CreateSplitXmlBinary = SplitXml.CreateSplitXml(parts); WordSampleDocInMemory = new MemoryStream(File.ReadAllBytes(@"../../../Files/demo.docx")); byte[] sampleXmlBinary = File.ReadAllBytes(@"../../../Files/split_demo.docx_20170227215840894.xml"); wordSplit.OpenAndSearchDocument(WordSampleDocInMemory, new MemoryStream(sampleXmlBinary)); MergeXmlBinary = WordSampleMerge.CreateMergeXml(); ErrorsCount = 0; WarningsCount = 0; }
public void Init() { var presentationSplit = new PresentationSplit("przykladowa-prezentacja"); PreSampleMerge = presentationSplit; DocSampleSplit = presentationSplit; SplitXml = presentationSplit; DocValidator = new OpenXmlValidator(); var parts = PartsSelectionTreeElementMock.GetListMock(); CreateSplitXmlBinary = SplitXml.CreateSplitXml(parts); PreSampleDocInMemory = new MemoryStream(File.ReadAllBytes(@"../../../Files/przykladowa-prezentacja.pptx")); byte[] sampleXmlBinary = File.ReadAllBytes(@"../../../Files/split_przykladowa-prezentacja.pptx_20170227215707619.xml"); presentationSplit.OpenAndSearchDocument(PreSampleDocInMemory, new MemoryStream(sampleXmlBinary)); MergeXmlBinary = PreSampleMerge.CreateMergeXml(); ErrorsCount = 0; WarningsCount = 0; }
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary> /// <param name="recommender">an item recommender</param> /// <param name="split">a positive-only feedback dataset split</param> /// <param name="test_users">a collection of integers with all test users</param> /// <param name="candidate_items">a collection of integers with all candidate items</param> /// <param name="candidate_item_mode">the mode used to determine the candidate items</param> /// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param> /// <param name="max_iter">the maximum number of iterations</param> /// <param name="find_iter">the report interval</param> /// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param> static public void DoRatingBasedRankingIterativeCrossValidation( this RatingPredictor recommender, ISplit <IRatings> split, IList <int> test_users, IList <int> candidate_items, CandidateItems candidate_item_mode, RepeatedEvents repeated_events, uint max_iter, uint find_iter = 1, bool show_fold_results = false) { if (!(recommender is IIterativeModel)) { throw new ArgumentException("recommender must be of type IIterativeModel"); } var split_recommenders = new RatingPredictor[split.NumberOfFolds]; var iterative_recommenders = new IIterativeModel[split.NumberOfFolds]; var fold_results = new ItemRecommendationEvaluationResults[split.NumberOfFolds]; // initial training and evaluation Parallel.For(0, (int)split.NumberOfFolds, i => { try { split_recommenders[i] = (RatingPredictor)recommender.Clone(); // to avoid changes in recommender split_recommenders[i].Ratings = split.Train[i]; split_recommenders[i].Train(); iterative_recommenders[i] = (IIterativeModel)split_recommenders[i]; var test_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Test[i]); var training_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Train[i]); fold_results[i] = Items.Evaluate(split_recommenders[i], test_data_posonly, training_data_posonly, test_users, candidate_items, candidate_item_mode, repeated_events); if (show_fold_results) { Console.WriteLine("fold {0} {1} iteration {2}", i, fold_results, iterative_recommenders[i].NumIter); } } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); Console.WriteLine("{0} iteration {1}", new ItemRecommendationEvaluationResults(fold_results), iterative_recommenders[0].NumIter); // iterative training and evaluation for (int it = (int)iterative_recommenders[0].NumIter + 1; it <= max_iter; it++) { Parallel.For(0, (int)split.NumberOfFolds, i => { try { iterative_recommenders[i].Iterate(); if (it % find_iter == 0) { var test_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Test[i]); var training_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Train[i]); fold_results[i] = Items.Evaluate(split_recommenders[i], test_data_posonly, training_data_posonly, test_users, candidate_items, candidate_item_mode, repeated_events); if (show_fold_results) { Console.WriteLine("fold {0} {1} iteration {2}", i, fold_results, it); } } } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); Console.WriteLine("{0} iteration {1}", new ItemRecommendationEvaluationResults(fold_results), it); } }
public NumberSplitCommandHandler(ISplit <NumberSplitCommand, int> split) : base(split) { }
public LLL(ISelect e1, ISplit f1) { Ee = e1; Ff = f1; }
public MultiplySplitCommandHandler(ISplit <MultiplySplitCommand, MultiplyItem> split) : base(split) { }
/// <summary>Find the the parameters resulting in the minimal results for a given evaluation measure</summary> /// <remarks>The recommender will be set to the best parameter value after calling this method.</remarks> /// <param name="evaluation_measure">the name of the evaluation measure</param> /// <param name="hp_names">the names of the hyperparameters to optimize</param> /// <param name="initial_hp_values">the values of the hyperparameters to try out first</param> /// <param name="recommender">the recommender</param> /// <param name="split">the dataset split to use</param> /// <returns>the best (lowest) average value for the hyperparameter</returns> public static double FindMinimum(string evaluation_measure, IList <string> hp_names, IList <Vector> initial_hp_values, RatingPredictor recommender, // TODO make more general? ISplit <IRatings> split) { var results = new Dictionary <string, double>(); var hp_vectors = new Dictionary <string, Vector>(); // initialize foreach (var hp_values in initial_hp_values) { string hp_string = CreateConfigString(hp_names, hp_values); results[hp_string] = Run(recommender, split, hp_string, evaluation_measure); hp_vectors[hp_string] = hp_values; } List <string> keys; for (int i = 0; i < num_it; i++) { if (results.Count != hp_vectors.Count) { throw new Exception(string.Format("{0} vs. {1}", results.Count, hp_vectors.Count)); } keys = new List <string>(results.Keys); keys.Sort(delegate(string k1, string k2) { return(results[k1].CompareTo(results[k2])); }); var min_key = keys.First(); var max_key = keys.Last(); Console.Error.WriteLine("Nelder-Mead: iteration {0} ({1})", i, results[min_key]); var worst_vector = hp_vectors[max_key]; var worst_result = results[max_key]; hp_vectors.Remove(max_key); results.Remove(max_key); // compute center var center = ComputeCenter(results, hp_vectors); // reflection //Console.Error.WriteLine("ref"); var reflection = center + alpha * (center - worst_vector); string ref_string = CreateConfigString(hp_names, reflection); double ref_result = Run(recommender, split, ref_string, evaluation_measure); if (results[min_key] <= ref_result && ref_result < results.Values.Max()) { results[ref_string] = ref_result; hp_vectors[ref_string] = reflection; continue; } // expansion if (ref_result < results[min_key]) { //Console.Error.WriteLine("exp"); var expansion = center + gamma * (center - worst_vector); string exp_string = CreateConfigString(hp_names, expansion); double exp_result = Run(recommender, split, exp_string, evaluation_measure); if (exp_result < ref_result) { results[exp_string] = exp_result; hp_vectors[exp_string] = expansion; } else { results[ref_string] = ref_result; hp_vectors[ref_string] = reflection; } continue; } // contraction //Console.Error.WriteLine("con"); var contraction = worst_vector + rho * (center - worst_vector); string con_string = CreateConfigString(hp_names, contraction); double con_result = Run(recommender, split, con_string, evaluation_measure); if (con_result < worst_result) { results[con_string] = con_result; hp_vectors[con_string] = contraction; continue; } // reduction //Console.Error.WriteLine("red"); var best_vector = hp_vectors[min_key]; var best_result = results[min_key]; hp_vectors.Remove(min_key); results.Remove(min_key); foreach (var key in new List <string>(results.Keys)) { var reduction = hp_vectors[key] + sigma * (hp_vectors[key] - best_vector); string red_string = CreateConfigString(hp_names, reduction); double red_result = Run(recommender, split, red_string, evaluation_measure); // replace by reduced vector results.Remove(key); hp_vectors.Remove(key); results[red_string] = red_result; hp_vectors[red_string] = reduction; } results[min_key] = best_result; hp_vectors[min_key] = best_vector; results[max_key] = worst_result; hp_vectors[max_key] = worst_vector; } keys = new List <string>(results.Keys); keys.Sort(delegate(string k1, string k2) { return(results[k1].CompareTo(results[k2])); }); // set to best hyperparameter values Recommender.Configure(recommender, keys.First()); return(results[keys.First()]); }
/// <summary>Evaluate on the folds of a dataset split</summary> /// <param name="recommender">an item recommender</param> /// <param name="split">a dataset split</param> /// <param name="test_users">a collection of integers with all test users</param> /// <param name="candidate_items">a collection of integers with all candidate items</param> /// <param name="candidate_item_mode">the mode used to determine the candidate items</param> /// <param name="compute_fit">if set to true measure fit on the training data as well</param> /// <param name="show_results">set to true to print results to STDERR</param> /// <returns>a dictionary containing the average results over the different folds of the split</returns> static public ItemRecommendationEvaluationResults DoCrossValidation( this IRecommender recommender, ISplit <IPosOnlyFeedback> split, IList <int> test_users, IList <int> candidate_items, CandidateItems candidate_item_mode = CandidateItems.OVERLAP, bool compute_fit = false, bool show_results = false) { var avg_results = new ItemRecommendationEvaluationResults(); if (!(recommender is ItemRecommender)) { throw new ArgumentException("recommender must be of type ItemRecommender"); } Parallel.For(0, (int)split.NumberOfFolds, fold => { try { var split_recommender = (ItemRecommender)recommender.Clone(); // avoid changes in recommender split_recommender.Feedback = split.Train[fold]; split_recommender.Train(); var fold_results = Items.Evaluate(split_recommender, split.Test[fold], split.Train[fold], test_users, candidate_items, candidate_item_mode); if (compute_fit) { fold_results["fit"] = (float)split_recommender.ComputeFit(); } // thread-safe stats lock (avg_results) foreach (var key in fold_results.Keys) { if (avg_results.ContainsKey(key)) { avg_results[key] += fold_results[key]; } else { avg_results[key] = fold_results[key]; } } if (show_results) { Console.Error.WriteLine("fold {0} {1}", fold, fold_results); } } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); foreach (var key in Items.Measures) { avg_results[key] /= split.NumberOfFolds; } avg_results["num_users"] /= split.NumberOfFolds; avg_results["num_items"] /= split.NumberOfFolds; return(avg_results); }
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary> /// <param name="recommender">a rating predictor</param> /// <param name="split">a rating dataset split</param> /// <param name="max_iter">the maximum number of iterations</param> /// <param name="find_iter">the report interval</param> /// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param> static public void DoIterativeCrossValidation( this RatingPredictor recommender, ISplit <IRatings> split, uint max_iter, uint find_iter = 1, bool show_fold_results = false) { if (!(recommender is IIterativeModel)) { throw new ArgumentException("recommender must be of type IIterativeModel"); } var split_recommenders = new RatingPredictor[split.NumberOfFolds]; var iterative_recommenders = new IIterativeModel[split.NumberOfFolds]; var fold_results = new RatingPredictionEvaluationResults[split.NumberOfFolds]; // initial training and evaluation Parallel.For(0, (int)split.NumberOfFolds, i => { try { split_recommenders[i] = (RatingPredictor)recommender.Clone(); // to avoid changes in recommender split_recommenders[i].Ratings = split.Train[i]; if (recommender is ITransductiveRatingPredictor) { ((ITransductiveRatingPredictor)split_recommenders[i]).AdditionalFeedback = split.Test[i]; } split_recommenders[i].Train(); iterative_recommenders[i] = (IIterativeModel)split_recommenders[i]; fold_results[i] = Ratings.Evaluate(split_recommenders[i], split.Test[i]); if (show_fold_results) { Console.Error.WriteLine("fold {0} {1} iteration {2}", i, fold_results[i], iterative_recommenders[i].NumIter); } } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); Console.WriteLine("{0} iteration {1}", new RatingPredictionEvaluationResults(fold_results), iterative_recommenders[0].NumIter); // iterative training and evaluation for (int it = (int)iterative_recommenders[0].NumIter + 1; it <= max_iter; it++) { Parallel.For(0, (int)split.NumberOfFolds, i => { try { iterative_recommenders[i].Iterate(); if (it % find_iter == 0) { fold_results[i] = Ratings.Evaluate(split_recommenders[i], split.Test[i]); if (show_fold_results) { Console.Error.WriteLine("fold {0} {1} iteration {2}", i, fold_results[i], it); } } } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); Console.WriteLine("{0} iteration {1}", new RatingPredictionEvaluationResults(fold_results), it); } }
public SplitCommandHandler(ISplit <TSplitCommand, TSplitResult> split) =>
/// <summary>Evaluate on the folds of a dataset split</summary> /// <param name="recommender">a rating predictor</param> /// <param name="split">a rating dataset split</param> /// <param name="show_results">set to true to print results to STDERR</param> /// <returns>a dictionary containing the average results over the different folds of the split</returns> public static Dictionary<string, double> EvaluateOnSplit(RatingPredictor recommender, ISplit<IRatings> split, bool show_results) { var avg_results = new Dictionary<string, double>(); foreach (var key in Measures) avg_results[key] = 0; for (int i = 0; i < split.NumberOfFolds; i++) { var split_recommender = (RatingPredictor) recommender.Clone(); // to avoid changes in recommender split_recommender.Ratings = split.Train[i]; split_recommender.Train(); var fold_results = Evaluate(split_recommender, split.Test[i]); foreach (var key in fold_results.Keys) avg_results[key] += fold_results[key]; if (show_results) Console.Error.WriteLine("fold {0}, RMSE {1,0:0.#####}, MAE {2,0:0.#####}", i, fold_results["RMSE"].ToString(CultureInfo.InvariantCulture), fold_results["MAE"].ToString(CultureInfo.InvariantCulture)); } foreach (var key in avg_results.Keys.ToList()) avg_results[key] /= split.NumberOfFolds; return avg_results; }
/// <summary>Find the the parameters resulting in the minimal results for a given evaluation measure (1D)</summary> /// <remarks>The recommender will be set to the best parameter value after calling this method.</remarks> /// <param name="evaluation_measure">the name of the evaluation measure</param> /// <param name="hp_name">the name of the hyperparameter to optimize</param> /// <param name="hp_values">the logarithms of the values of the hyperparameter to try out</param> /// <param name="basis">the basis to use for the logarithms</param> /// <param name="recommender">the recommender</param> /// <param name="split">the dataset split to use</param> /// <returns>the best (lowest) average value for the hyperparameter</returns> public static double FindMinimumExponential( string evaluation_measure, string hp_name, double[] hp_values, double basis, RatingPrediction.RatingPredictor recommender, ISplit<IRatings> split) { var new_hp_values = new double[hp_values.Length]; for (int i = 0; i < hp_values.Length; i++) new_hp_values[i] = Math.Pow(basis, hp_values[i]); return FindMinimum(evaluation_measure, hp_name, new_hp_values, recommender, split); }
/// <summary>Evaluate on the folds of a dataset split</summary> /// <param name="recommender">an item recommender</param> /// <param name="split">a dataset split</param> /// <param name="candidate_items">a collection of integers with all candidate items</param> /// <param name="candidate_item_mode">the mode used to determine the candidate items</param> /// <param name="compute_fit">if set to true measure fit on the training data as well</param> /// <param name="show_results">set to true to print results to STDERR</param> /// <returns>a dictionary containing the average results over the different folds of the split</returns> static public EvaluationResults DoRatingBasedRankingCrossValidation( this RatingPredictor recommender, ISplit <IRatings> split, IList <int> candidate_items, CandidateItems candidate_item_mode = CandidateItems.OVERLAP, bool compute_fit = false, bool show_results = false) { var avg_results = new ItemRecommendationEvaluationResults(); Parallel.For(0, (int)split.NumberOfFolds, fold => { try { var split_recommender = (RatingPredictor)recommender.Clone(); // avoid changes in recommender split_recommender.Ratings = split.Train[fold]; split_recommender.Train(); var test_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Test[fold]); var training_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Train[fold]); IList <int> test_users = test_data_posonly.AllUsers; var fold_results = Items.Evaluate(split_recommender, test_data_posonly, training_data_posonly, test_users, candidate_items, candidate_item_mode); if (compute_fit) { fold_results["fit"] = (float)split_recommender.ComputeFit(); } // thread-safe stats lock (avg_results) foreach (var key in fold_results.Keys) { if (avg_results.ContainsKey(key)) { avg_results[key] += fold_results[key]; } else { avg_results[key] = fold_results[key]; } } if (show_results) { Console.Error.WriteLine("fold {0} {1}", fold, fold_results); } } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); foreach (var key in Items.Measures) { avg_results[key] /= split.NumberOfFolds; } avg_results["num_users"] /= split.NumberOfFolds; avg_results["num_items"] /= split.NumberOfFolds; return(avg_results); }
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary> /// <param name="recommender">a rating predictor</param> /// <param name="split">a rating dataset split</param> /// <param name="max_iter">the maximum number of iterations</param> /// <param name="find_iter">the report interval</param> /// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param> public static void DoIterativeCrossValidation( this RatingPredictor recommender, ISplit<IRatings> split, int max_iter, int find_iter = 1, bool show_fold_results = false) { if (!(recommender is IIterativeModel)) throw new ArgumentException("recommender must be of type IIterativeModel"); var split_recommenders = new RatingPredictor[split.NumberOfFolds]; var iterative_recommenders = new IIterativeModel[split.NumberOfFolds]; var fold_results = new RatingPredictionEvaluationResults[split.NumberOfFolds]; // initial training and evaluation Parallel.For(0, (int) split.NumberOfFolds, i => { try { split_recommenders[i] = (RatingPredictor) recommender.Clone(); // to avoid changes in recommender split_recommenders[i].Ratings = split.Train[i]; if (recommender is ITransductiveRatingPredictor) ((ITransductiveRatingPredictor) split_recommenders[i]).AdditionalFeedback = split.Test[i]; split_recommenders[i].Train(); iterative_recommenders[i] = (IIterativeModel) split_recommenders[i]; fold_results[i] = Ratings.Evaluate(split_recommenders[i], split.Test[i]); if (show_fold_results) Console.Error.WriteLine("fold {0} {1} iteration {2}", i, fold_results[i], iterative_recommenders[i].NumIter); } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw e; } }); Console.WriteLine("{0} iteration {1}", new RatingPredictionEvaluationResults(fold_results), iterative_recommenders[0].NumIter); // iterative training and evaluation for (int it = (int) iterative_recommenders[0].NumIter + 1; it <= max_iter; it++) { Parallel.For(0, (int) split.NumberOfFolds, i => { try { iterative_recommenders[i].Iterate(); if (it % find_iter == 0) { fold_results[i] = Ratings.Evaluate(split_recommenders[i], split.Test[i]); if (show_fold_results) Console.Error.WriteLine("fold {0} {1} iteration {2}", i, fold_results[i], it); } } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw e; } }); Console.WriteLine("{0} iteration {1}", new RatingPredictionEvaluationResults(fold_results), it); } }
/// <summary>Evaluate on the folds of a dataset split</summary> /// <param name="recommender">a rating predictor</param> /// <param name="split">a rating dataset split</param> /// <returns>a dictionary containing the average results over the different folds of the split</returns> static public Dictionary <string, double> EvaluateOnSplit(RatingPredictor recommender, ISplit <IRatings> split) { return(EvaluateOnSplit(recommender, split, false)); }
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary> /// <param name="recommender">an item recommender</param> /// <param name="split">a positive-only feedback dataset split</param> /// <param name="test_users">a collection of integers with all test users</param> /// <param name="candidate_items">a collection of integers with all candidate items</param> /// <param name="candidate_item_mode">the mode used to determine the candidate items</param> /// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param> /// <param name="max_iter">the maximum number of iterations</param> /// <param name="find_iter">the report interval</param> /// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param> public static void DoIterativeCrossValidation( this IRecommender recommender, ISplit<IPosOnlyFeedback> split, IList<int> test_users, IList<int> candidate_items, CandidateItems candidate_item_mode, RepeatedEvents repeated_events, uint max_iter, uint find_iter = 1, bool show_fold_results = false) { if (!(recommender is IIterativeModel)) throw new ArgumentException("recommender must be of type IIterativeModel"); if (!(recommender is ItemRecommender)) throw new ArgumentException("recommender must be of type ItemRecommender"); var split_recommenders = new ItemRecommender[split.NumberOfFolds]; var iterative_recommenders = new IIterativeModel[split.NumberOfFolds]; var fold_results = new ItemRecommendationEvaluationResults[split.NumberOfFolds]; // initial training and evaluation Parallel.For(0, (int) split.NumberOfFolds, i => { try { split_recommenders[i] = (ItemRecommender) recommender.Clone(); // to avoid changes in recommender split_recommenders[i].Feedback = split.Train[i]; split_recommenders[i].Train(); iterative_recommenders[i] = (IIterativeModel) split_recommenders[i]; fold_results[i] = Items.Evaluate(split_recommenders[i], split.Test[i], split.Train[i], test_users, candidate_items, candidate_item_mode, repeated_events); if (show_fold_results) Console.WriteLine("fold {0} {1} iteration {2}", i, fold_results, iterative_recommenders[i].NumIter); } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); Console.WriteLine("{0} iteration {1}", new ItemRecommendationEvaluationResults(fold_results), iterative_recommenders[0].NumIter); // iterative training and evaluation for (int it = (int) iterative_recommenders[0].NumIter + 1; it <= max_iter; it++) { Parallel.For(0, (int) split.NumberOfFolds, i => { try { iterative_recommenders[i].Iterate(); if (it % find_iter == 0) { fold_results[i] = Items.Evaluate(split_recommenders[i], split.Test[i], split.Train[i], test_users, candidate_items, candidate_item_mode, repeated_events); if (show_fold_results) Console.WriteLine("fold {0} {1} iteration {2}", i, fold_results, it); } } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); Console.WriteLine("{0} iteration {1}", new ItemRecommendationEvaluationResults(fold_results), it); } }
/// <summary>Evaluate on the folds of a dataset split</summary> /// <param name="recommender">a rating predictor</param> /// <param name="split">a rating dataset split</param> /// <param name="show_results">set to true to print results to STDERR</param> /// <returns>a dictionary containing the average results over the different folds of the split</returns> static public Dictionary <string, double> EvaluateOnSplit(RatingPredictor recommender, ISplit <IRatings> split, bool show_results) { var avg_results = new Dictionary <string, double>(); foreach (var key in Measures) { avg_results[key] = 0; } for (int i = 0; i < split.NumberOfFolds; i++) { var split_recommender = (RatingPredictor)recommender.Clone(); // to avoid changes in recommender split_recommender.Ratings = split.Train[i]; split_recommender.Train(); var fold_results = Evaluate(split_recommender, split.Test[i]); foreach (var key in fold_results.Keys) { avg_results[key] += fold_results[key]; } if (show_results) { Console.Error.WriteLine("fold {0}, RMSE {1,0:0.#####}, MAE {2,0:0.#####}", i, fold_results["RMSE"].ToString(CultureInfo.InvariantCulture), fold_results["MAE"].ToString(CultureInfo.InvariantCulture)); } } foreach (var key in avg_results.Keys.ToList()) { avg_results[key] /= split.NumberOfFolds; } return(avg_results); }
/// <summary>Find the the parameters resulting in the minimal results for a given evaluation measure</summary> /// <remarks>The recommender will be set to the best parameter value after calling this method.</remarks> /// <param name="evaluation_measure">the name of the evaluation measure</param> /// <param name="hp_names">the names of the hyperparameters to optimize</param> /// <param name="initial_hp_values">the values of the hyperparameters to try out first</param> /// <param name="recommender">the recommender</param> /// <param name="split">the dataset split to use</param> /// <returns>the best (lowest) average value for the hyperparameter</returns> public static double FindMinimum(string evaluation_measure, IList<string> hp_names, IList<Vector> initial_hp_values, RatingPredictor recommender, // TODO make more general? ISplit<IRatings> split) { var results = new Dictionary<string, double>(); var hp_vectors = new Dictionary<string, Vector>(); // initialize foreach (var hp_values in initial_hp_values) { string hp_string = CreateConfigString(hp_names, hp_values); results[hp_string] = Run(recommender, split, hp_string, evaluation_measure); hp_vectors[hp_string] = hp_values; } List<string> keys; for (int i = 0; i < num_it; i++) { if (results.Count != hp_vectors.Count) throw new Exception(string.Format("{0} vs. {1}", results.Count, hp_vectors.Count)); keys = new List<string>(results.Keys); keys.Sort(delegate(string k1, string k2) { return results[k1].CompareTo(results[k2]); }); var min_key = keys.First(); var max_key = keys.Last(); Console.Error.WriteLine("Nelder-Mead: iteration {0} ({1})", i, results[min_key]); var worst_vector = hp_vectors[max_key]; var worst_result = results[max_key]; hp_vectors.Remove(max_key); results.Remove(max_key); // compute center var center = ComputeCenter(results, hp_vectors); // reflection //Console.Error.WriteLine("ref"); var reflection = center + alpha * (center - worst_vector); string ref_string = CreateConfigString(hp_names, reflection); double ref_result = Run(recommender, split, ref_string, evaluation_measure); if (results[min_key] <= ref_result && ref_result < results.Values.Max()) { results[ref_string] = ref_result; hp_vectors[ref_string] = reflection; continue; } // expansion if (ref_result < results[min_key]) { //Console.Error.WriteLine("exp"); var expansion = center + gamma * (center - worst_vector); string exp_string = CreateConfigString(hp_names, expansion); double exp_result = Run(recommender, split, exp_string, evaluation_measure); if (exp_result < ref_result) { results[exp_string] = exp_result; hp_vectors[exp_string] = expansion; } else { results[ref_string] = ref_result; hp_vectors[ref_string] = reflection; } continue; } // contraction //Console.Error.WriteLine("con"); var contraction = worst_vector + rho * (center - worst_vector); string con_string = CreateConfigString(hp_names, contraction); double con_result = Run(recommender, split, con_string, evaluation_measure); if (con_result < worst_result) { results[con_string] = con_result; hp_vectors[con_string] = contraction; continue; } // reduction //Console.Error.WriteLine("red"); var best_vector = hp_vectors[min_key]; var best_result = results[min_key]; hp_vectors.Remove(min_key); results.Remove(min_key); foreach (var key in new List<string>(results.Keys)) { var reduction = hp_vectors[key] + sigma * (hp_vectors[key] - best_vector); string red_string = CreateConfigString(hp_names, reduction); double red_result = Run(recommender, split, red_string, evaluation_measure); // replace by reduced vector results.Remove(key); hp_vectors.Remove(key); results[red_string] = red_result; hp_vectors[red_string] = reduction; } results[min_key] = best_result; hp_vectors[min_key] = best_vector; results[max_key] = worst_result; hp_vectors[max_key] = worst_vector; } keys = new List<string>(results.Keys); keys.Sort(delegate(string k1, string k2) { return results[k1].CompareTo(results[k2]); }); // set to best hyperparameter values Recommender.Configure(recommender, keys.First()); return results[keys.First()]; }
/// <summary>Evaluate on the folds of a dataset split</summary> /// <param name="recommender">a rating predictor</param> /// <param name="split">a rating dataset split</param> /// <returns>a dictionary containing the average results over the different folds of the split</returns> public static Dictionary<string, double> EvaluateOnSplit(RatingPredictor recommender, ISplit<IRatings> split) { return EvaluateOnSplit(recommender, split, false); }