private double Evaluate(string hp_string)
{
recommender.Configure(hp_string);
double result = recommender.DoCrossValidation(split)[evaluation_measure];
Console.Error.WriteLine("Nelder-Mead: {0}: {1}", hp_string, result.ToString(CultureInfo.InvariantCulture));
return(result);
}
static double Run(RatingPredictor recommender, ISplit <IRatings> split, string hp_string, string evaluation_measure)
{
recommender.Configure(hp_string);
double result = recommender.DoCrossValidation(split)[evaluation_measure];
Console.Error.WriteLine("Nelder-Mead: {0}: {1}", hp_string, result.ToString(CultureInfo.InvariantCulture));
return(result);
}
public static void Main(string[] args)
{
// TODO add random seed
// TODO report per-user times
string data_file = args[0];
string method = args[1];
string options = args[2];
int num_test_users = int.Parse(args[3]);
// load the data
var all_data = RatingData.Read(data_file);
// TODO randomize
var test_users = new HashSet <int>(Enumerable.Range(0, num_test_users));
var update_indices = new List <int>();
var eval_indices = new List <int>();
foreach (int user_id in test_users)
{
if (all_data.ByUser[user_id].Count > 1)
{
var user_indices = all_data.ByUser[user_id];
for (int i = 0; i < user_indices.Count - 1; i++)
{
update_indices.Add(user_indices[i]);
}
for (int i = user_indices.Count - 1; i < user_indices.Count; i++)
{
eval_indices.Add(user_indices[i]);
}
}
}
var training_indices = new List <int>();
for (int i = 0; i < all_data.Count; i++)
{
if (!test_users.Contains(all_data.Users[i]))
{
training_indices.Add(i);
}
}
var training_data = new MyMediaLite.Data.Ratings();
foreach (int i in training_indices)
{
training_data.Add(all_data.Users[i], all_data.Items[i], all_data[i]);
}
var update_data = new RatingsProxy(all_data, update_indices);
var eval_data = new RatingsProxy(all_data, eval_indices);
Console.Write(training_data.Statistics());
Console.Write(update_data.Statistics());
Console.Write(eval_data.Statistics());
// prepare recommender
RatingPredictor recommender = method.CreateRatingPredictor();
recommender.Configure(options);
recommender.Ratings = training_data;
Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "ratings range: [{0}, {1}]", recommender.MinRating, recommender.MaxRating));
Console.WriteLine("recommender: {0}", recommender);
recommender.Train();
// I. complete retraining
Console.WriteLine(
"complete training: {0}",
recommender.EvaluateFoldInCompleteRetraining(update_data, eval_data));
// II. online updates
Console.WriteLine(
"incremental training: {0}",
((IncrementalRatingPredictor)recommender).EvaluateFoldInIncrementalTraining(update_data, eval_data));
// III. fold-in
Console.WriteLine(
"fold-in: {0}",
((IFoldInRatingPredictor)recommender).EvaluateFoldIn(update_data, eval_data));
}
/// <summary>Find the the parameters resulting in the minimal results for a given evaluation measure</summary>
/// <remarks>The recommender will be set to the best parameter value after calling this method.</remarks>
/// <param name="evaluation_measure">the name of the evaluation measure</param>
/// <param name="hp_names">the names of the hyperparameters to optimize</param>
/// <param name="initial_hp_values">the values of the hyperparameters to try out first</param>
/// <param name="recommender">the recommender</param>
/// <param name="split">the dataset split to use</param>
/// <returns>the best (lowest) average value for the hyperparameter</returns>
public static double FindMinimum(
string evaluation_measure,
IList <string> hp_names,
IList <DenseVector> initial_hp_values,
RatingPredictor recommender, // TODO make more general?
ISplit <IRatings> split)
{
var results = new Dictionary <string, double>();
var hp_vectors = new Dictionary <string, DenseVector>();
// initialize
foreach (var hp_values in initial_hp_values)
{
string hp_string = CreateConfigString(hp_names, hp_values.ToArray());
results[hp_string] = Run(recommender, split, hp_string, evaluation_measure);
hp_vectors[hp_string] = hp_values;
}
List <string> keys;
for (int i = 0; i < num_it; i++)
{
if (results.Count != hp_vectors.Count)
{
throw new Exception(string.Format("{0} vs. {1}", results.Count, hp_vectors.Count));
}
keys = new List <string>(results.Keys);
keys.Sort(delegate(string k1, string k2) { return(results[k1].CompareTo(results[k2])); });
var min_key = keys.First();
var max_key = keys.Last();
Console.Error.WriteLine(string.Format(CultureInfo.InvariantCulture, "Nelder-Mead: iteration {0} ({1})", i, results[min_key]));
var worst_vector = hp_vectors[max_key];
var worst_result = results[max_key];
hp_vectors.Remove(max_key);
results.Remove(max_key);
// compute center
var center = ComputeCenter(results, hp_vectors);
// reflection
//Console.Error.WriteLine("ref");
var reflection = center + alpha * (center - worst_vector);
string ref_string = CreateConfigString(hp_names, reflection.ToArray());
// .ToArray() is necessary to compile with Visual Studio
double ref_result = Run(recommender, split, ref_string, evaluation_measure);
if (results[min_key] <= ref_result && ref_result < results.Values.Max())
{
results[ref_string] = ref_result;
hp_vectors[ref_string] = reflection;
continue;
}
// expansion
if (ref_result < results[min_key])
{
//Console.Error.WriteLine("exp");
var expansion = center + gamma * (center - worst_vector);
string exp_string = CreateConfigString(hp_names, expansion.ToArray());
double exp_result = Run(recommender, split, exp_string, evaluation_measure);
if (exp_result < ref_result)
{
results[exp_string] = exp_result;
hp_vectors[exp_string] = expansion;
}
else
{
results[ref_string] = ref_result;
hp_vectors[ref_string] = reflection;
}
continue;
}
// contraction
//Console.Error.WriteLine("con");
var contraction = worst_vector + rho * (center - worst_vector);
string con_string = CreateConfigString(hp_names, contraction.ToArray());
double con_result = Run(recommender, split, con_string, evaluation_measure);
if (con_result < worst_result)
{
results[con_string] = con_result;
hp_vectors[con_string] = contraction;
continue;
}
// reduction
//Console.Error.WriteLine("red");
var best_vector = hp_vectors[min_key];
var best_result = results[min_key];
hp_vectors.Remove(min_key);
results.Remove(min_key);
foreach (var key in new List <string>(results.Keys))
{
var reduction = hp_vectors[key] + sigma * (hp_vectors[key] - best_vector);
string red_string = CreateConfigString(hp_names, reduction.ToArray());
double red_result = Run(recommender, split, red_string, evaluation_measure);
// replace by reduced vector
results.Remove(key);
hp_vectors.Remove(key);
results[red_string] = red_result;
hp_vectors[red_string] = reduction;
}
results[min_key] = best_result;
hp_vectors[min_key] = best_vector;
results[max_key] = worst_result;
hp_vectors[max_key] = worst_vector;
}
keys = new List <string>(results.Keys);
keys.Sort(delegate(string k1, string k2) { return(results[k1].CompareTo(results[k2])); });
// set to best hyperparameter values
recommender.Configure(keys.First());
return(results[keys.First()]);
}
