private double Evaluate(string hp_string) { recommender.Configure(hp_string); double result = recommender.DoCrossValidation(split)[evaluation_measure]; Console.Error.WriteLine("Nelder-Mead: {0}: {1}", hp_string, result.ToString(CultureInfo.InvariantCulture)); return(result); }
static double Run(RatingPredictor recommender, ISplit <IRatings> split, string hp_string, string evaluation_measure) { recommender.Configure(hp_string); double result = recommender.DoCrossValidation(split)[evaluation_measure]; Console.Error.WriteLine("Nelder-Mead: {0}: {1}", hp_string, result.ToString(CultureInfo.InvariantCulture)); return(result); }
public static void Main(string[] args) { // TODO add random seed // TODO report per-user times string data_file = args[0]; string method = args[1]; string options = args[2]; int num_test_users = int.Parse(args[3]); // load the data var all_data = RatingData.Read(data_file); // TODO randomize var test_users = new HashSet <int>(Enumerable.Range(0, num_test_users)); var update_indices = new List <int>(); var eval_indices = new List <int>(); foreach (int user_id in test_users) { if (all_data.ByUser[user_id].Count > 1) { var user_indices = all_data.ByUser[user_id]; for (int i = 0; i < user_indices.Count - 1; i++) { update_indices.Add(user_indices[i]); } for (int i = user_indices.Count - 1; i < user_indices.Count; i++) { eval_indices.Add(user_indices[i]); } } } var training_indices = new List <int>(); for (int i = 0; i < all_data.Count; i++) { if (!test_users.Contains(all_data.Users[i])) { training_indices.Add(i); } } var training_data = new MyMediaLite.Data.Ratings(); foreach (int i in training_indices) { training_data.Add(all_data.Users[i], all_data.Items[i], all_data[i]); } var update_data = new RatingsProxy(all_data, update_indices); var eval_data = new RatingsProxy(all_data, eval_indices); Console.Write(training_data.Statistics()); Console.Write(update_data.Statistics()); Console.Write(eval_data.Statistics()); // prepare recommender RatingPredictor recommender = method.CreateRatingPredictor(); recommender.Configure(options); recommender.Ratings = training_data; Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "ratings range: [{0}, {1}]", recommender.MinRating, recommender.MaxRating)); Console.WriteLine("recommender: {0}", recommender); recommender.Train(); // I. complete retraining Console.WriteLine( "complete training: {0}", recommender.EvaluateFoldInCompleteRetraining(update_data, eval_data)); // II. online updates Console.WriteLine( "incremental training: {0}", ((IncrementalRatingPredictor)recommender).EvaluateFoldInIncrementalTraining(update_data, eval_data)); // III. fold-in Console.WriteLine( "fold-in: {0}", ((IFoldInRatingPredictor)recommender).EvaluateFoldIn(update_data, eval_data)); }
/// <summary>Find the the parameters resulting in the minimal results for a given evaluation measure</summary> /// <remarks>The recommender will be set to the best parameter value after calling this method.</remarks> /// <param name="evaluation_measure">the name of the evaluation measure</param> /// <param name="hp_names">the names of the hyperparameters to optimize</param> /// <param name="initial_hp_values">the values of the hyperparameters to try out first</param> /// <param name="recommender">the recommender</param> /// <param name="split">the dataset split to use</param> /// <returns>the best (lowest) average value for the hyperparameter</returns> public static double FindMinimum( string evaluation_measure, IList <string> hp_names, IList <DenseVector> initial_hp_values, RatingPredictor recommender, // TODO make more general? ISplit <IRatings> split) { var results = new Dictionary <string, double>(); var hp_vectors = new Dictionary <string, DenseVector>(); // initialize foreach (var hp_values in initial_hp_values) { string hp_string = CreateConfigString(hp_names, hp_values.ToArray()); results[hp_string] = Run(recommender, split, hp_string, evaluation_measure); hp_vectors[hp_string] = hp_values; } List <string> keys; for (int i = 0; i < num_it; i++) { if (results.Count != hp_vectors.Count) { throw new Exception(string.Format("{0} vs. {1}", results.Count, hp_vectors.Count)); } keys = new List <string>(results.Keys); keys.Sort(delegate(string k1, string k2) { return(results[k1].CompareTo(results[k2])); }); var min_key = keys.First(); var max_key = keys.Last(); Console.Error.WriteLine(string.Format(CultureInfo.InvariantCulture, "Nelder-Mead: iteration {0} ({1})", i, results[min_key])); var worst_vector = hp_vectors[max_key]; var worst_result = results[max_key]; hp_vectors.Remove(max_key); results.Remove(max_key); // compute center var center = ComputeCenter(results, hp_vectors); // reflection //Console.Error.WriteLine("ref"); var reflection = center + alpha * (center - worst_vector); string ref_string = CreateConfigString(hp_names, reflection.ToArray()); // .ToArray() is necessary to compile with Visual Studio double ref_result = Run(recommender, split, ref_string, evaluation_measure); if (results[min_key] <= ref_result && ref_result < results.Values.Max()) { results[ref_string] = ref_result; hp_vectors[ref_string] = reflection; continue; } // expansion if (ref_result < results[min_key]) { //Console.Error.WriteLine("exp"); var expansion = center + gamma * (center - worst_vector); string exp_string = CreateConfigString(hp_names, expansion.ToArray()); double exp_result = Run(recommender, split, exp_string, evaluation_measure); if (exp_result < ref_result) { results[exp_string] = exp_result; hp_vectors[exp_string] = expansion; } else { results[ref_string] = ref_result; hp_vectors[ref_string] = reflection; } continue; } // contraction //Console.Error.WriteLine("con"); var contraction = worst_vector + rho * (center - worst_vector); string con_string = CreateConfigString(hp_names, contraction.ToArray()); double con_result = Run(recommender, split, con_string, evaluation_measure); if (con_result < worst_result) { results[con_string] = con_result; hp_vectors[con_string] = contraction; continue; } // reduction //Console.Error.WriteLine("red"); var best_vector = hp_vectors[min_key]; var best_result = results[min_key]; hp_vectors.Remove(min_key); results.Remove(min_key); foreach (var key in new List <string>(results.Keys)) { var reduction = hp_vectors[key] + sigma * (hp_vectors[key] - best_vector); string red_string = CreateConfigString(hp_names, reduction.ToArray()); double red_result = Run(recommender, split, red_string, evaluation_measure); // replace by reduced vector results.Remove(key); hp_vectors.Remove(key); results[red_string] = red_result; hp_vectors[red_string] = reduction; } results[min_key] = best_result; hp_vectors[min_key] = best_vector; results[max_key] = worst_result; hp_vectors[max_key] = worst_vector; } keys = new List <string>(results.Keys); keys.Sort(delegate(string k1, string k2) { return(results[k1].CompareTo(results[k2])); }); // set to best hyperparameter values recommender.Configure(keys.First()); return(results[keys.First()]); }