/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">a rating predictor</param>
/// <param name="split">a rating dataset split</param>
/// <param name="compute_fit">if set to true measure fit on the training data as well</param>
/// <param name="show_fold_results">set to true to print per-fold results to STDERR</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
public static RatingPredictionEvaluationResults DoCrossValidation(
this RatingPredictor recommender,
ISplit<IRatings> split,
bool compute_fit = false,
bool show_fold_results = false)
{
var fold_results = new RatingPredictionEvaluationResults[split.NumberOfFolds];
Parallel.For(0, (int) split.NumberOfFolds, i =>
{
try
{
var split_recommender = (RatingPredictor) recommender.Clone(); // to avoid changes in recommender
split_recommender.Ratings = split.Train[i];
if (recommender is ITransductiveRatingPredictor)
((ITransductiveRatingPredictor) split_recommender).AdditionalFeedback = split.Test[i];
split_recommender.Train();
fold_results[i] = Ratings.Evaluate(split_recommender, split.Test[i]);
if (compute_fit)
fold_results[i]["fit"] = (float) split_recommender.ComputeFit();
if (show_fold_results)
Console.Error.WriteLine("fold {0} {1}", i, fold_results[i]);
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw e;
}
});
return new RatingPredictionEvaluationResults(fold_results);
}
/// <summary>Find the the parameters resulting in the minimal results for a given evaluation measure (2D)</summary>
/// <remarks>The recommender will be set to the best parameter value after calling this method.</remarks>
/// <param name="evaluation_measure">the name of the evaluation measure</param>
/// <param name="hp_name1">the name of the first hyperparameter to optimize</param>
/// <param name="hp_values1">the values of the first hyperparameter to try out</param>
/// <param name="hp_name2">the name of the second hyperparameter to optimize</param>
/// <param name="hp_values2">the values of the second hyperparameter to try out</param>
/// <param name="recommender">the recommender</param>
/// <param name="split">the dataset split to use</param>
/// <returns>the best (lowest) average value for the hyperparameter</returns>
public static double FindMinimum(
string evaluation_measure,
string hp_name1, string hp_name2,
double[] hp_values1, double[] hp_values2,
RatingPredictor recommender,
ISplit <IRatings> split)
{
double min_result = double.MaxValue;
int min_i = -1;
int min_j = -1;
for (int i = 0; i < hp_values1.Length; i++)
{
for (int j = 0; j < hp_values2.Length; j++)
{
recommender.SetProperty(hp_name1, hp_values1[i].ToString(CultureInfo.InvariantCulture));
recommender.SetProperty(hp_name2, hp_values2[j].ToString(CultureInfo.InvariantCulture));
Console.Error.WriteLine("reg_u={0} reg_i={1}", hp_values1[i].ToString(CultureInfo.InvariantCulture), hp_values2[j].ToString(CultureInfo.InvariantCulture)); // TODO this is not generic
double result = recommender.DoCrossValidation(split)[evaluation_measure];
if (result < min_result)
{
min_i = i;
min_j = j;
min_result = result;
}
}
}
// set to best hyperparameter values
recommender.SetProperty(hp_name1, hp_values1[min_i].ToString(CultureInfo.InvariantCulture));
recommender.SetProperty(hp_name2, hp_values2[min_j].ToString(CultureInfo.InvariantCulture));
return(min_result);
}
/// <summary>Find the the parameters resulting in the minimal results for a given evaluation measure (1D)</summary>
/// <remarks>The recommender will be set to the best parameter value after calling this method.</remarks>
/// <param name="evaluation_measure">the name of the evaluation measure</param>
/// <param name="hyperparameter_name">the name of the hyperparameter to optimize</param>
/// <param name="hyperparameter_values">the values of the hyperparameter to try out</param>
/// <param name="recommender">the recommender</param>
/// <param name="split">the dataset split to use</param>
/// <returns>the best (lowest) average value for the hyperparameter</returns>
public static double FindMinimum(
string evaluation_measure,
string hyperparameter_name,
double[] hyperparameter_values,
RatingPredictor recommender,
ISplit<IRatings> split)
{
double min_result = double.MaxValue;
int min_i = -1;
for (int i = 0; i < hyperparameter_values.Length; i++)
{
recommender.SetProperty(hyperparameter_name, hyperparameter_values[i].ToString(CultureInfo.InvariantCulture));
double result = recommender.DoCrossValidation(split)[evaluation_measure];
if (result < min_result)
{
min_i = i;
min_result = result;
}
}
recommender.SetProperty(hyperparameter_name, hyperparameter_values[min_i].ToString(CultureInfo.InvariantCulture));
return min_result;
}
public Cube(byte[] redArray, byte[] greenArray, byte[] blueArray, ISelect select, ISplit split)
{
ColorCount = redArray.Length;
Initialize(redArray, greenArray, blueArray);
SelectType = select;
SplitType = split;
}
/// <summary>Find the the parameters resulting in the minimal results for a given evaluation measure (2D)</summary>
/// <remarks>The recommender will be set to the best parameter value after calling this method.</remarks>
/// <param name="evaluation_measure">the name of the evaluation measure</param>
/// <param name="hp_name1">the name of the first hyperparameter to optimize</param>
/// <param name="hp_values1">the values of the first hyperparameter to try out</param>
/// <param name="hp_name2">the name of the second hyperparameter to optimize</param>
/// <param name="hp_values2">the values of the second hyperparameter to try out</param>
/// <param name="recommender">the recommender</param>
/// <param name="split">the dataset split to use</param>
/// <returns>the best (lowest) average value for the hyperparameter</returns>
public static double FindMinimum(
string evaluation_measure,
string hp_name1, string hp_name2,
double[] hp_values1, double[] hp_values2,
RatingPredictor recommender,
ISplit<IRatings> split)
{
double min_result = double.MaxValue;
int min_i = -1;
int min_j = -1;
for (int i = 0; i < hp_values1.Length; i++)
for (int j = 0; j < hp_values2.Length; j++)
{
recommender.SetProperty(hp_name1, hp_values1[i].ToString(CultureInfo.InvariantCulture));
recommender.SetProperty(hp_name2, hp_values2[j].ToString(CultureInfo.InvariantCulture));
Console.Error.WriteLine("reg_u={0} reg_i={1}", hp_values1[i].ToString(CultureInfo.InvariantCulture), hp_values2[j].ToString(CultureInfo.InvariantCulture)); // TODO this is not generic
double result = recommender.DoCrossValidation(split)[evaluation_measure];
if (result < min_result)
{
min_i = i;
min_j = j;
min_result = result;
}
}
// set to best hyperparameter values
recommender.SetProperty(hp_name1, hp_values1[min_i].ToString(CultureInfo.InvariantCulture));
recommender.SetProperty(hp_name2, hp_values2[min_j].ToString(CultureInfo.InvariantCulture));
return min_result;
}
/// <summary>Find the the parameters resulting in the minimal results for a given evaluation measure (1D)</summary>
/// <remarks>The recommender will be set to the best parameter value after calling this method.</remarks>
/// <param name="evaluation_measure">the name of the evaluation measure</param>
/// <param name="hyperparameter_name">the name of the hyperparameter to optimize</param>
/// <param name="hyperparameter_values">the values of the hyperparameter to try out</param>
/// <param name="recommender">the recommender</param>
/// <param name="split">the dataset split to use</param>
/// <returns>the best (lowest) average value for the hyperparameter</returns>
public static double FindMinimum(
string evaluation_measure,
string hyperparameter_name,
double[] hyperparameter_values,
RatingPredictor recommender,
ISplit <IRatings> split)
{
double min_result = double.MaxValue;
int min_i = -1;
for (int i = 0; i < hyperparameter_values.Length; i++)
{
recommender.SetProperty(hyperparameter_name, hyperparameter_values[i].ToString(CultureInfo.InvariantCulture));
double result = recommender.DoCrossValidation(split)[evaluation_measure];
if (result < min_result)
{
min_i = i;
min_result = result;
}
}
recommender.SetProperty(hyperparameter_name, hyperparameter_values[min_i].ToString(CultureInfo.InvariantCulture));
return(min_result);
}
private void Init()
{
this.split = new RatingsSimpleSplit(recommender.Ratings, split_ratio);
//this.split = new RatingCrossValidationSplit(recommender.Ratings, 5);
// TODO manage this via reflection?
if (recommender is UserItemBaseline)
{
this.hp_names = new string[] { "reg_u", "reg_i" };
this.initial_hp_values = CreateInitialValues(
new double[][] {
new double[] { 25, 10 },
new double[] { 10, 25 },
new double[] { 2, 5 },
new double[] { 5, 2 },
new double[] { 1, 4 },
new double[] { 4, 1 },
new double[] { 3, 3 },
}
);
}
else if (recommender is BiasedMatrixFactorization)
{
this.hp_names = new string[] { "regularization", "bias_reg" };
this.initial_hp_values = CreateInitialValues(
// TODO reg_u and reg_i (in a second step?)
new double[][]
{
new double[] { 0.1, 0 },
new double[] { 0.01, 0 },
new double[] { 0.0001, 0 },
new double[] { 0.00001, 0 },
new double[] { 0.1, 0.0001 },
new double[] { 0.01, 0.0001 },
new double[] { 0.0001, 0.0001 },
new double[] { 0.00001, 0.0001 },
}
);
}
else if (recommender is MatrixFactorization)
{
this.hp_names = new string[] { "regularization" };
// TODO normal interval search could be more efficient
this.initial_hp_values = CreateInitialValues(
new double[][]
{
new double[] { 0.1 },
new double[] { 0.01 },
new double[] { 0.0001 },
new double[] { 0.00001 },
}
);
}
// TODO kNN-based methods
else
{
throw new Exception("not prepared for type " + recommender.GetType().ToString());
}
}
static double Run(RatingPredictor recommender, ISplit<IRatings> split, string hp_string, string evaluation_measure)
{
recommender.Configure(hp_string);
double result = recommender.DoCrossValidation(split)[evaluation_measure];
Console.Error.WriteLine("Nelder-Mead: {0}: {1}", hp_string, result.ToString(CultureInfo.InvariantCulture));
return result;
}
static double Run(RatingPredictor recommender, ISplit <IRatings> split, string hp_string, string evaluation_measure)
{
Recommender.Configure(recommender, hp_string);
double result = Eval.Ratings.EvaluateOnSplit(recommender, split)[evaluation_measure];
Console.Error.WriteLine("Nelder-Mead: {0}: {1}", hp_string, result.ToString(CultureInfo.InvariantCulture));
return(result);
}
/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="split">a dataset split</param>
/// <param name="test_users">a collection of integers with all test users</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="compute_fit">if set to true measure fit on the training data as well</param>
/// <param name="show_results">set to true to print results to STDERR</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
public static ItemRecommendationEvaluationResults DoCrossValidation(
this IRecommender recommender,
ISplit<IPosOnlyFeedback> split,
IList<int> test_users,
IList<int> candidate_items,
CandidateItems candidate_item_mode = CandidateItems.OVERLAP,
bool compute_fit = false,
bool show_results = false)
{
var avg_results = new ItemRecommendationEvaluationResults();
if (!(recommender is ItemRecommender))
throw new ArgumentException("recommender must be of type ItemRecommender");
Parallel.For(0, (int) split.NumberOfFolds, fold =>
{
try
{
var split_recommender = (ItemRecommender) recommender.Clone(); // avoid changes in recommender
split_recommender.Feedback = split.Train[fold];
split_recommender.Train();
var fold_results = Items.Evaluate(split_recommender, split.Test[fold], split.Train[fold], test_users, candidate_items, candidate_item_mode);
if (compute_fit)
fold_results["fit"] = (float) split_recommender.ComputeFit();
// thread-safe stats
lock (avg_results)
foreach (var key in fold_results.Keys)
if (avg_results.ContainsKey(key))
avg_results[key] += fold_results[key];
else
avg_results[key] = fold_results[key];
if (show_results)
Console.Error.WriteLine("fold {0} {1}", fold, fold_results);
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
foreach (var key in Items.Measures)
avg_results[key] /= split.NumberOfFolds;
avg_results["num_users"] /= split.NumberOfFolds;
avg_results["num_items"] /= split.NumberOfFolds;
if (compute_fit)
avg_results["fit"] /= split.NumberOfFolds;
return avg_results;
}
/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="split">a dataset split</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="compute_fit">if set to true measure fit on the training data as well</param>
/// <param name="show_results">set to true to print results to STDERR</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
static public EvaluationResults DoRatingBasedRankingCrossValidation(
this RatingPredictor recommender,
ISplit<IRatings> split,
IList<int> candidate_items,
CandidateItems candidate_item_mode = CandidateItems.OVERLAP,
bool compute_fit = false,
bool show_results = false)
{
var avg_results = new ItemRecommendationEvaluationResults();
Parallel.For(0, (int) split.NumberOfFolds, fold =>
{
try
{
var split_recommender = (RatingPredictor) recommender.Clone(); // avoid changes in recommender
split_recommender.Ratings = split.Train[fold];
split_recommender.Train();
var test_data_posonly = new PosOnlyFeedback<SparseBooleanMatrix>(split.Test[fold]);
var training_data_posonly = new PosOnlyFeedback<SparseBooleanMatrix>(split.Train[fold]);
IList<int> test_users = test_data_posonly.AllUsers;
var fold_results = Items.Evaluate(split_recommender, test_data_posonly, training_data_posonly, test_users, candidate_items, candidate_item_mode);
if (compute_fit)
fold_results["fit"] = (float) split_recommender.ComputeFit();
// thread-safe stats
lock (avg_results)
foreach (var key in fold_results.Keys)
if (avg_results.ContainsKey(key))
avg_results[key] += fold_results[key];
else
avg_results[key] = fold_results[key];
if (show_results)
Console.Error.WriteLine("fold {0} {1}", fold, fold_results);
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
foreach (var key in Items.Measures)
avg_results[key] /= split.NumberOfFolds;
avg_results["num_users"] /= split.NumberOfFolds;
avg_results["num_items"] /= split.NumberOfFolds;
return avg_results;
}
/// <summary>Find the the parameters resulting in the minimal results for a given evaluation measure (1D)</summary>
/// <remarks>The recommender will be set to the best parameter value after calling this method.</remarks>
/// <param name="evaluation_measure">the name of the evaluation measure</param>
/// <param name="hp_name">the name of the hyperparameter to optimize</param>
/// <param name="hp_values">the logarithms of the values of the hyperparameter to try out</param>
/// <param name="basis">the basis to use for the logarithms</param>
/// <param name="recommender">the recommender</param>
/// <param name="split">the dataset split to use</param>
/// <returns>the best (lowest) average value for the hyperparameter</returns>
public static double FindMinimumExponential(string evaluation_measure,
string hp_name,
double[] hp_values,
double basis,
RatingPrediction.RatingPredictor recommender,
ISplit <IRatings> split)
{
var new_hp_values = new double[hp_values.Length];
for (int i = 0; i < hp_values.Length; i++)
{
new_hp_values[i] = Math.Pow(basis, hp_values[i]);
}
return(FindMinimum(evaluation_measure, hp_name, new_hp_values, recommender, split));
}
public Cube(byte[] pixels, ISelect select, ISplit split)
{
ColorCount = pixels.Length / 4;
SelectType = select;
SplitType = split;
var redArray = new byte[ColorCount];
var greenArray = new byte[ColorCount];
var blueArray = new byte[ColorCount];
for (int i = 0; i < ColorCount; i++)
{
redArray[i] = pixels[i * 4 + 2];
greenArray[i] = pixels[i * 4 + 1];
blueArray[i] = pixels[i * 4];
}
Initialize(redArray, greenArray, blueArray);
}
/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">a rating predictor</param>
/// <param name="split">a rating dataset split</param>
/// <param name="compute_fit">if set to true measure fit on the training data as well</param>
/// <param name="show_results">set to true to print results to STDERR</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
public static RatingPredictionEvaluationResults DoCrossValidation(
this RatingPredictor recommender,
ISplit<IRatings> split,
bool compute_fit = false,
bool show_results = false)
{
var avg_results = new RatingPredictionEvaluationResults();
Parallel.For(0, (int) split.NumberOfFolds, i =>
{
try
{
var split_recommender = (RatingPredictor) recommender.Clone(); // to avoid changes in recommender
split_recommender.Ratings = split.Train[i];
if (recommender is ITransductiveRatingPredictor)
((ITransductiveRatingPredictor) split_recommender).AdditionalFeedback = split.Test[i];
split_recommender.Train();
var fold_results = Ratings.Evaluate(split_recommender, split.Test[i]);
if (compute_fit)
fold_results["fit"] = (float) split_recommender.ComputeFit();
// thread-safe stats
lock (avg_results)
foreach (var key in fold_results.Keys)
if (avg_results.ContainsKey(key))
avg_results[key] += fold_results[key];
else
avg_results[key] = fold_results[key];
if (show_results)
Console.Error.WriteLine("fold {0} {1}", i, fold_results);
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw e;
}
});
foreach (var key in Ratings.Measures)
avg_results[key] /= split.NumberOfFolds;
return avg_results;
}
/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">a rating predictor</param>
/// <param name="split">a rating dataset split</param>
/// <param name="compute_fit">if set to true measure fit on the training data as well</param>
/// <param name="show_fold_results">set to true to print per-fold results to STDERR</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
static public RatingPredictionEvaluationResults DoCrossValidation(
this RatingPredictor recommender,
ISplit <IRatings> split,
bool compute_fit = false,
bool show_fold_results = false)
{
var fold_results = new RatingPredictionEvaluationResults[split.NumberOfFolds];
Parallel.For(0, (int)split.NumberOfFolds, i =>
{
try
{
var split_recommender = (RatingPredictor)recommender.Clone(); // to avoid changes in recommender
split_recommender.Ratings = split.Train[i];
if (recommender is ITransductiveRatingPredictor)
{
((ITransductiveRatingPredictor)split_recommender).AdditionalFeedback = split.Test[i];
}
split_recommender.Train();
fold_results[i] = Ratings.Evaluate(split_recommender, split.Test[i]);
if (compute_fit)
{
fold_results[i]["fit"] = (float)split_recommender.ComputeFit();
}
if (show_fold_results)
{
Console.Error.WriteLine("fold {0} {1}", i, fold_results[i]);
}
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
return(new RatingPredictionEvaluationResults(fold_results));
}
public void Init()
{
var excelSplit = new ExcelSplit("ExcelTutorialR1 — edytowalny");
ExcSampleSplit = excelSplit;
SplitXml = excelSplit;
ExcSampleMerge = excelSplit;
DocValidator = new OpenXmlValidator();
var parts = PartsSelectionTreeElementMock.GetListMock();
CreateSplitXmlBinary = SplitXml.CreateSplitXml(parts);
ExcSampleDocInMemory = new MemoryStream(File.ReadAllBytes(@"../../../Files/ExcelTutorialR1 — edytowalny.xlsx"));
byte[] sampleXmlBinary = File.ReadAllBytes(@"../../../Files/split_ExcelTutorialR1 — edytowalny.xlsx_20170304230639351.xml");
excelSplit.OpenAndSearchDocument(ExcSampleDocInMemory, new MemoryStream(sampleXmlBinary));
MergeXmlBinary = ExcSampleMerge.CreateMergeXml();
ErrorsCount = 0;
WarningsCount = 0;
}
public void Init()
{
var wordSplit = new WordSplit("demo");
WordSampleMerge = wordSplit;
WordSampleSplit = wordSplit;
SplitXml = wordSplit;
DocValidator = new OpenXmlValidator();
var parts = PartsSelectionTreeElementMock.GetListMock();
CreateSplitXmlBinary = SplitXml.CreateSplitXml(parts);
WordSampleDocInMemory = new MemoryStream(File.ReadAllBytes(@"../../../Files/demo.docx"));
byte[] sampleXmlBinary = File.ReadAllBytes(@"../../../Files/split_demo.docx_20170227215840894.xml");
wordSplit.OpenAndSearchDocument(WordSampleDocInMemory, new MemoryStream(sampleXmlBinary));
MergeXmlBinary = WordSampleMerge.CreateMergeXml();
ErrorsCount = 0;
WarningsCount = 0;
}
public void Init()
{
var presentationSplit = new PresentationSplit("przykladowa-prezentacja");
PreSampleMerge = presentationSplit;
DocSampleSplit = presentationSplit;
SplitXml = presentationSplit;
DocValidator = new OpenXmlValidator();
var parts = PartsSelectionTreeElementMock.GetListMock();
CreateSplitXmlBinary = SplitXml.CreateSplitXml(parts);
PreSampleDocInMemory = new MemoryStream(File.ReadAllBytes(@"../../../Files/przykladowa-prezentacja.pptx"));
byte[] sampleXmlBinary = File.ReadAllBytes(@"../../../Files/split_przykladowa-prezentacja.pptx_20170227215707619.xml");
presentationSplit.OpenAndSearchDocument(PreSampleDocInMemory, new MemoryStream(sampleXmlBinary));
MergeXmlBinary = PreSampleMerge.CreateMergeXml();
ErrorsCount = 0;
WarningsCount = 0;
}
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="split">a positive-only feedback dataset split</param>
/// <param name="test_users">a collection of integers with all test users</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param>
/// <param name="max_iter">the maximum number of iterations</param>
/// <param name="find_iter">the report interval</param>
/// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param>
static public void DoRatingBasedRankingIterativeCrossValidation(
this RatingPredictor recommender,
ISplit <IRatings> split,
IList <int> test_users,
IList <int> candidate_items,
CandidateItems candidate_item_mode,
RepeatedEvents repeated_events,
uint max_iter,
uint find_iter = 1,
bool show_fold_results = false)
{
if (!(recommender is IIterativeModel))
{
throw new ArgumentException("recommender must be of type IIterativeModel");
}
var split_recommenders = new RatingPredictor[split.NumberOfFolds];
var iterative_recommenders = new IIterativeModel[split.NumberOfFolds];
var fold_results = new ItemRecommendationEvaluationResults[split.NumberOfFolds];
// initial training and evaluation
Parallel.For(0, (int)split.NumberOfFolds, i =>
{
try
{
split_recommenders[i] = (RatingPredictor)recommender.Clone(); // to avoid changes in recommender
split_recommenders[i].Ratings = split.Train[i];
split_recommenders[i].Train();
iterative_recommenders[i] = (IIterativeModel)split_recommenders[i];
var test_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Test[i]);
var training_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Train[i]);
fold_results[i] = Items.Evaluate(split_recommenders[i], test_data_posonly, training_data_posonly, test_users, candidate_items, candidate_item_mode, repeated_events);
if (show_fold_results)
{
Console.WriteLine("fold {0} {1} iteration {2}", i, fold_results, iterative_recommenders[i].NumIter);
}
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
Console.WriteLine("{0} iteration {1}", new ItemRecommendationEvaluationResults(fold_results), iterative_recommenders[0].NumIter);
// iterative training and evaluation
for (int it = (int)iterative_recommenders[0].NumIter + 1; it <= max_iter; it++)
{
Parallel.For(0, (int)split.NumberOfFolds, i =>
{
try
{
iterative_recommenders[i].Iterate();
if (it % find_iter == 0)
{
var test_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Test[i]);
var training_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Train[i]);
fold_results[i] = Items.Evaluate(split_recommenders[i], test_data_posonly, training_data_posonly, test_users, candidate_items, candidate_item_mode, repeated_events);
if (show_fold_results)
{
Console.WriteLine("fold {0} {1} iteration {2}", i, fold_results, it);
}
}
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
Console.WriteLine("{0} iteration {1}", new ItemRecommendationEvaluationResults(fold_results), it);
}
}
public NumberSplitCommandHandler(ISplit <NumberSplitCommand, int> split)
: base(split)
{
}
public LLL(ISelect e1, ISplit f1)
{
Ee = e1;
Ff = f1;
}
public MultiplySplitCommandHandler(ISplit <MultiplySplitCommand, MultiplyItem> split)
: base(split)
{
}
/// <summary>Find the the parameters resulting in the minimal results for a given evaluation measure</summary>
/// <remarks>The recommender will be set to the best parameter value after calling this method.</remarks>
/// <param name="evaluation_measure">the name of the evaluation measure</param>
/// <param name="hp_names">the names of the hyperparameters to optimize</param>
/// <param name="initial_hp_values">the values of the hyperparameters to try out first</param>
/// <param name="recommender">the recommender</param>
/// <param name="split">the dataset split to use</param>
/// <returns>the best (lowest) average value for the hyperparameter</returns>
public static double FindMinimum(string evaluation_measure,
IList <string> hp_names,
IList <Vector> initial_hp_values,
RatingPredictor recommender, // TODO make more general?
ISplit <IRatings> split)
{
var results = new Dictionary <string, double>();
var hp_vectors = new Dictionary <string, Vector>();
// initialize
foreach (var hp_values in initial_hp_values)
{
string hp_string = CreateConfigString(hp_names, hp_values);
results[hp_string] = Run(recommender, split, hp_string, evaluation_measure);
hp_vectors[hp_string] = hp_values;
}
List <string> keys;
for (int i = 0; i < num_it; i++)
{
if (results.Count != hp_vectors.Count)
{
throw new Exception(string.Format("{0} vs. {1}", results.Count, hp_vectors.Count));
}
keys = new List <string>(results.Keys);
keys.Sort(delegate(string k1, string k2) { return(results[k1].CompareTo(results[k2])); });
var min_key = keys.First();
var max_key = keys.Last();
Console.Error.WriteLine("Nelder-Mead: iteration {0} ({1})", i, results[min_key]);
var worst_vector = hp_vectors[max_key];
var worst_result = results[max_key];
hp_vectors.Remove(max_key);
results.Remove(max_key);
// compute center
var center = ComputeCenter(results, hp_vectors);
// reflection
//Console.Error.WriteLine("ref");
var reflection = center + alpha * (center - worst_vector);
string ref_string = CreateConfigString(hp_names, reflection);
double ref_result = Run(recommender, split, ref_string, evaluation_measure);
if (results[min_key] <= ref_result && ref_result < results.Values.Max())
{
results[ref_string] = ref_result;
hp_vectors[ref_string] = reflection;
continue;
}
// expansion
if (ref_result < results[min_key])
{
//Console.Error.WriteLine("exp");
var expansion = center + gamma * (center - worst_vector);
string exp_string = CreateConfigString(hp_names, expansion);
double exp_result = Run(recommender, split, exp_string, evaluation_measure);
if (exp_result < ref_result)
{
results[exp_string] = exp_result;
hp_vectors[exp_string] = expansion;
}
else
{
results[ref_string] = ref_result;
hp_vectors[ref_string] = reflection;
}
continue;
}
// contraction
//Console.Error.WriteLine("con");
var contraction = worst_vector + rho * (center - worst_vector);
string con_string = CreateConfigString(hp_names, contraction);
double con_result = Run(recommender, split, con_string, evaluation_measure);
if (con_result < worst_result)
{
results[con_string] = con_result;
hp_vectors[con_string] = contraction;
continue;
}
// reduction
//Console.Error.WriteLine("red");
var best_vector = hp_vectors[min_key];
var best_result = results[min_key];
hp_vectors.Remove(min_key);
results.Remove(min_key);
foreach (var key in new List <string>(results.Keys))
{
var reduction = hp_vectors[key] + sigma * (hp_vectors[key] - best_vector);
string red_string = CreateConfigString(hp_names, reduction);
double red_result = Run(recommender, split, red_string, evaluation_measure);
// replace by reduced vector
results.Remove(key);
hp_vectors.Remove(key);
results[red_string] = red_result;
hp_vectors[red_string] = reduction;
}
results[min_key] = best_result;
hp_vectors[min_key] = best_vector;
results[max_key] = worst_result;
hp_vectors[max_key] = worst_vector;
}
keys = new List <string>(results.Keys);
keys.Sort(delegate(string k1, string k2) { return(results[k1].CompareTo(results[k2])); });
// set to best hyperparameter values
Recommender.Configure(recommender, keys.First());
return(results[keys.First()]);
}
/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="split">a dataset split</param>
/// <param name="test_users">a collection of integers with all test users</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="compute_fit">if set to true measure fit on the training data as well</param>
/// <param name="show_results">set to true to print results to STDERR</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
static public ItemRecommendationEvaluationResults DoCrossValidation(
this IRecommender recommender,
ISplit <IPosOnlyFeedback> split,
IList <int> test_users,
IList <int> candidate_items,
CandidateItems candidate_item_mode = CandidateItems.OVERLAP,
bool compute_fit = false,
bool show_results = false)
{
var avg_results = new ItemRecommendationEvaluationResults();
if (!(recommender is ItemRecommender))
{
throw new ArgumentException("recommender must be of type ItemRecommender");
}
Parallel.For(0, (int)split.NumberOfFolds, fold =>
{
try
{
var split_recommender = (ItemRecommender)recommender.Clone(); // avoid changes in recommender
split_recommender.Feedback = split.Train[fold];
split_recommender.Train();
var fold_results = Items.Evaluate(split_recommender, split.Test[fold], split.Train[fold], test_users, candidate_items, candidate_item_mode);
if (compute_fit)
{
fold_results["fit"] = (float)split_recommender.ComputeFit();
}
// thread-safe stats
lock (avg_results)
foreach (var key in fold_results.Keys)
{
if (avg_results.ContainsKey(key))
{
avg_results[key] += fold_results[key];
}
else
{
avg_results[key] = fold_results[key];
}
}
if (show_results)
{
Console.Error.WriteLine("fold {0} {1}", fold, fold_results);
}
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
foreach (var key in Items.Measures)
{
avg_results[key] /= split.NumberOfFolds;
}
avg_results["num_users"] /= split.NumberOfFolds;
avg_results["num_items"] /= split.NumberOfFolds;
return(avg_results);
}
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary>
/// <param name="recommender">a rating predictor</param>
/// <param name="split">a rating dataset split</param>
/// <param name="max_iter">the maximum number of iterations</param>
/// <param name="find_iter">the report interval</param>
/// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param>
static public void DoIterativeCrossValidation(
this RatingPredictor recommender,
ISplit <IRatings> split,
uint max_iter,
uint find_iter = 1,
bool show_fold_results = false)
{
if (!(recommender is IIterativeModel))
{
throw new ArgumentException("recommender must be of type IIterativeModel");
}
var split_recommenders = new RatingPredictor[split.NumberOfFolds];
var iterative_recommenders = new IIterativeModel[split.NumberOfFolds];
var fold_results = new RatingPredictionEvaluationResults[split.NumberOfFolds];
// initial training and evaluation
Parallel.For(0, (int)split.NumberOfFolds, i =>
{
try
{
split_recommenders[i] = (RatingPredictor)recommender.Clone(); // to avoid changes in recommender
split_recommenders[i].Ratings = split.Train[i];
if (recommender is ITransductiveRatingPredictor)
{
((ITransductiveRatingPredictor)split_recommenders[i]).AdditionalFeedback = split.Test[i];
}
split_recommenders[i].Train();
iterative_recommenders[i] = (IIterativeModel)split_recommenders[i];
fold_results[i] = Ratings.Evaluate(split_recommenders[i], split.Test[i]);
if (show_fold_results)
{
Console.Error.WriteLine("fold {0} {1} iteration {2}", i, fold_results[i], iterative_recommenders[i].NumIter);
}
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
Console.WriteLine("{0} iteration {1}", new RatingPredictionEvaluationResults(fold_results), iterative_recommenders[0].NumIter);
// iterative training and evaluation
for (int it = (int)iterative_recommenders[0].NumIter + 1; it <= max_iter; it++)
{
Parallel.For(0, (int)split.NumberOfFolds, i =>
{
try
{
iterative_recommenders[i].Iterate();
if (it % find_iter == 0)
{
fold_results[i] = Ratings.Evaluate(split_recommenders[i], split.Test[i]);
if (show_fold_results)
{
Console.Error.WriteLine("fold {0} {1} iteration {2}", i, fold_results[i], it);
}
}
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
Console.WriteLine("{0} iteration {1}", new RatingPredictionEvaluationResults(fold_results), it);
}
}
public SplitCommandHandler(ISplit <TSplitCommand, TSplitResult> split) =>
/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">a rating predictor</param>
/// <param name="split">a rating dataset split</param>
/// <param name="show_results">set to true to print results to STDERR</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
public static Dictionary<string, double> EvaluateOnSplit(RatingPredictor recommender, ISplit<IRatings> split, bool show_results)
{
var avg_results = new Dictionary<string, double>();
foreach (var key in Measures)
avg_results[key] = 0;
for (int i = 0; i < split.NumberOfFolds; i++)
{
var split_recommender = (RatingPredictor) recommender.Clone(); // to avoid changes in recommender
split_recommender.Ratings = split.Train[i];
split_recommender.Train();
var fold_results = Evaluate(split_recommender, split.Test[i]);
foreach (var key in fold_results.Keys)
avg_results[key] += fold_results[key];
if (show_results)
Console.Error.WriteLine("fold {0}, RMSE {1,0:0.#####}, MAE {2,0:0.#####}", i, fold_results["RMSE"].ToString(CultureInfo.InvariantCulture), fold_results["MAE"].ToString(CultureInfo.InvariantCulture));
}
foreach (var key in avg_results.Keys.ToList())
avg_results[key] /= split.NumberOfFolds;
return avg_results;
}
/// <summary>Find the the parameters resulting in the minimal results for a given evaluation measure (1D)</summary>
/// <remarks>The recommender will be set to the best parameter value after calling this method.</remarks>
/// <param name="evaluation_measure">the name of the evaluation measure</param>
/// <param name="hp_name">the name of the hyperparameter to optimize</param>
/// <param name="hp_values">the logarithms of the values of the hyperparameter to try out</param>
/// <param name="basis">the basis to use for the logarithms</param>
/// <param name="recommender">the recommender</param>
/// <param name="split">the dataset split to use</param>
/// <returns>the best (lowest) average value for the hyperparameter</returns>
public static double FindMinimumExponential(
string evaluation_measure,
string hp_name,
double[] hp_values,
double basis,
RatingPrediction.RatingPredictor recommender,
ISplit<IRatings> split)
{
var new_hp_values = new double[hp_values.Length];
for (int i = 0; i < hp_values.Length; i++)
new_hp_values[i] = Math.Pow(basis, hp_values[i]);
return FindMinimum(evaluation_measure, hp_name, new_hp_values, recommender, split);
}
/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="split">a dataset split</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="compute_fit">if set to true measure fit on the training data as well</param>
/// <param name="show_results">set to true to print results to STDERR</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
static public EvaluationResults DoRatingBasedRankingCrossValidation(
this RatingPredictor recommender,
ISplit <IRatings> split,
IList <int> candidate_items,
CandidateItems candidate_item_mode = CandidateItems.OVERLAP,
bool compute_fit = false,
bool show_results = false)
{
var avg_results = new ItemRecommendationEvaluationResults();
Parallel.For(0, (int)split.NumberOfFolds, fold =>
{
try
{
var split_recommender = (RatingPredictor)recommender.Clone(); // avoid changes in recommender
split_recommender.Ratings = split.Train[fold];
split_recommender.Train();
var test_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Test[fold]);
var training_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Train[fold]);
IList <int> test_users = test_data_posonly.AllUsers;
var fold_results = Items.Evaluate(split_recommender, test_data_posonly, training_data_posonly, test_users, candidate_items, candidate_item_mode);
if (compute_fit)
{
fold_results["fit"] = (float)split_recommender.ComputeFit();
}
// thread-safe stats
lock (avg_results)
foreach (var key in fold_results.Keys)
{
if (avg_results.ContainsKey(key))
{
avg_results[key] += fold_results[key];
}
else
{
avg_results[key] = fold_results[key];
}
}
if (show_results)
{
Console.Error.WriteLine("fold {0} {1}", fold, fold_results);
}
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
foreach (var key in Items.Measures)
{
avg_results[key] /= split.NumberOfFolds;
}
avg_results["num_users"] /= split.NumberOfFolds;
avg_results["num_items"] /= split.NumberOfFolds;
return(avg_results);
}
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary>
/// <param name="recommender">a rating predictor</param>
/// <param name="split">a rating dataset split</param>
/// <param name="max_iter">the maximum number of iterations</param>
/// <param name="find_iter">the report interval</param>
/// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param>
public static void DoIterativeCrossValidation(
this RatingPredictor recommender,
ISplit<IRatings> split,
int max_iter,
int find_iter = 1,
bool show_fold_results = false)
{
if (!(recommender is IIterativeModel))
throw new ArgumentException("recommender must be of type IIterativeModel");
var split_recommenders = new RatingPredictor[split.NumberOfFolds];
var iterative_recommenders = new IIterativeModel[split.NumberOfFolds];
var fold_results = new RatingPredictionEvaluationResults[split.NumberOfFolds];
// initial training and evaluation
Parallel.For(0, (int) split.NumberOfFolds, i =>
{
try
{
split_recommenders[i] = (RatingPredictor) recommender.Clone(); // to avoid changes in recommender
split_recommenders[i].Ratings = split.Train[i];
if (recommender is ITransductiveRatingPredictor)
((ITransductiveRatingPredictor) split_recommenders[i]).AdditionalFeedback = split.Test[i];
split_recommenders[i].Train();
iterative_recommenders[i] = (IIterativeModel) split_recommenders[i];
fold_results[i] = Ratings.Evaluate(split_recommenders[i], split.Test[i]);
if (show_fold_results)
Console.Error.WriteLine("fold {0} {1} iteration {2}", i, fold_results[i], iterative_recommenders[i].NumIter);
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw e;
}
});
Console.WriteLine("{0} iteration {1}", new RatingPredictionEvaluationResults(fold_results), iterative_recommenders[0].NumIter);
// iterative training and evaluation
for (int it = (int) iterative_recommenders[0].NumIter + 1; it <= max_iter; it++)
{
Parallel.For(0, (int) split.NumberOfFolds, i =>
{
try
{
iterative_recommenders[i].Iterate();
if (it % find_iter == 0)
{
fold_results[i] = Ratings.Evaluate(split_recommenders[i], split.Test[i]);
if (show_fold_results)
Console.Error.WriteLine("fold {0} {1} iteration {2}", i, fold_results[i], it);
}
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw e;
}
});
Console.WriteLine("{0} iteration {1}", new RatingPredictionEvaluationResults(fold_results), it);
}
}
/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">a rating predictor</param>
/// <param name="split">a rating dataset split</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
static public Dictionary <string, double> EvaluateOnSplit(RatingPredictor recommender, ISplit <IRatings> split)
{
return(EvaluateOnSplit(recommender, split, false));
}
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="split">a positive-only feedback dataset split</param>
/// <param name="test_users">a collection of integers with all test users</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param>
/// <param name="max_iter">the maximum number of iterations</param>
/// <param name="find_iter">the report interval</param>
/// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param>
public static void DoIterativeCrossValidation(
this IRecommender recommender,
ISplit<IPosOnlyFeedback> split,
IList<int> test_users,
IList<int> candidate_items,
CandidateItems candidate_item_mode,
RepeatedEvents repeated_events,
uint max_iter,
uint find_iter = 1,
bool show_fold_results = false)
{
if (!(recommender is IIterativeModel))
throw new ArgumentException("recommender must be of type IIterativeModel");
if (!(recommender is ItemRecommender))
throw new ArgumentException("recommender must be of type ItemRecommender");
var split_recommenders = new ItemRecommender[split.NumberOfFolds];
var iterative_recommenders = new IIterativeModel[split.NumberOfFolds];
var fold_results = new ItemRecommendationEvaluationResults[split.NumberOfFolds];
// initial training and evaluation
Parallel.For(0, (int) split.NumberOfFolds, i =>
{
try
{
split_recommenders[i] = (ItemRecommender) recommender.Clone(); // to avoid changes in recommender
split_recommenders[i].Feedback = split.Train[i];
split_recommenders[i].Train();
iterative_recommenders[i] = (IIterativeModel) split_recommenders[i];
fold_results[i] = Items.Evaluate(split_recommenders[i], split.Test[i], split.Train[i], test_users, candidate_items, candidate_item_mode, repeated_events);
if (show_fold_results)
Console.WriteLine("fold {0} {1} iteration {2}", i, fold_results, iterative_recommenders[i].NumIter);
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
Console.WriteLine("{0} iteration {1}", new ItemRecommendationEvaluationResults(fold_results), iterative_recommenders[0].NumIter);
// iterative training and evaluation
for (int it = (int) iterative_recommenders[0].NumIter + 1; it <= max_iter; it++)
{
Parallel.For(0, (int) split.NumberOfFolds, i =>
{
try
{
iterative_recommenders[i].Iterate();
if (it % find_iter == 0)
{
fold_results[i] = Items.Evaluate(split_recommenders[i], split.Test[i], split.Train[i], test_users, candidate_items, candidate_item_mode, repeated_events);
if (show_fold_results)
Console.WriteLine("fold {0} {1} iteration {2}", i, fold_results, it);
}
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
Console.WriteLine("{0} iteration {1}", new ItemRecommendationEvaluationResults(fold_results), it);
}
}
/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">a rating predictor</param>
/// <param name="split">a rating dataset split</param>
/// <param name="show_results">set to true to print results to STDERR</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
static public Dictionary <string, double> EvaluateOnSplit(RatingPredictor recommender, ISplit <IRatings> split, bool show_results)
{
var avg_results = new Dictionary <string, double>();
foreach (var key in Measures)
{
avg_results[key] = 0;
}
for (int i = 0; i < split.NumberOfFolds; i++)
{
var split_recommender = (RatingPredictor)recommender.Clone(); // to avoid changes in recommender
split_recommender.Ratings = split.Train[i];
split_recommender.Train();
var fold_results = Evaluate(split_recommender, split.Test[i]);
foreach (var key in fold_results.Keys)
{
avg_results[key] += fold_results[key];
}
if (show_results)
{
Console.Error.WriteLine("fold {0}, RMSE {1,0:0.#####}, MAE {2,0:0.#####}", i, fold_results["RMSE"].ToString(CultureInfo.InvariantCulture), fold_results["MAE"].ToString(CultureInfo.InvariantCulture));
}
}
foreach (var key in avg_results.Keys.ToList())
{
avg_results[key] /= split.NumberOfFolds;
}
return(avg_results);
}
/// <summary>Find the the parameters resulting in the minimal results for a given evaluation measure</summary>
/// <remarks>The recommender will be set to the best parameter value after calling this method.</remarks>
/// <param name="evaluation_measure">the name of the evaluation measure</param>
/// <param name="hp_names">the names of the hyperparameters to optimize</param>
/// <param name="initial_hp_values">the values of the hyperparameters to try out first</param>
/// <param name="recommender">the recommender</param>
/// <param name="split">the dataset split to use</param>
/// <returns>the best (lowest) average value for the hyperparameter</returns>
public static double FindMinimum(string evaluation_measure,
IList<string> hp_names,
IList<Vector> initial_hp_values,
RatingPredictor recommender, // TODO make more general?
ISplit<IRatings> split)
{
var results = new Dictionary<string, double>();
var hp_vectors = new Dictionary<string, Vector>();
// initialize
foreach (var hp_values in initial_hp_values)
{
string hp_string = CreateConfigString(hp_names, hp_values);
results[hp_string] = Run(recommender, split, hp_string, evaluation_measure);
hp_vectors[hp_string] = hp_values;
}
List<string> keys;
for (int i = 0; i < num_it; i++)
{
if (results.Count != hp_vectors.Count)
throw new Exception(string.Format("{0} vs. {1}", results.Count, hp_vectors.Count));
keys = new List<string>(results.Keys);
keys.Sort(delegate(string k1, string k2) { return results[k1].CompareTo(results[k2]); });
var min_key = keys.First();
var max_key = keys.Last();
Console.Error.WriteLine("Nelder-Mead: iteration {0} ({1})", i, results[min_key]);
var worst_vector = hp_vectors[max_key];
var worst_result = results[max_key];
hp_vectors.Remove(max_key);
results.Remove(max_key);
// compute center
var center = ComputeCenter(results, hp_vectors);
// reflection
//Console.Error.WriteLine("ref");
var reflection = center + alpha * (center - worst_vector);
string ref_string = CreateConfigString(hp_names, reflection);
double ref_result = Run(recommender, split, ref_string, evaluation_measure);
if (results[min_key] <= ref_result && ref_result < results.Values.Max())
{
results[ref_string] = ref_result;
hp_vectors[ref_string] = reflection;
continue;
}
// expansion
if (ref_result < results[min_key])
{
//Console.Error.WriteLine("exp");
var expansion = center + gamma * (center - worst_vector);
string exp_string = CreateConfigString(hp_names, expansion);
double exp_result = Run(recommender, split, exp_string, evaluation_measure);
if (exp_result < ref_result)
{
results[exp_string] = exp_result;
hp_vectors[exp_string] = expansion;
}
else
{
results[ref_string] = ref_result;
hp_vectors[ref_string] = reflection;
}
continue;
}
// contraction
//Console.Error.WriteLine("con");
var contraction = worst_vector + rho * (center - worst_vector);
string con_string = CreateConfigString(hp_names, contraction);
double con_result = Run(recommender, split, con_string, evaluation_measure);
if (con_result < worst_result)
{
results[con_string] = con_result;
hp_vectors[con_string] = contraction;
continue;
}
// reduction
//Console.Error.WriteLine("red");
var best_vector = hp_vectors[min_key];
var best_result = results[min_key];
hp_vectors.Remove(min_key);
results.Remove(min_key);
foreach (var key in new List<string>(results.Keys))
{
var reduction = hp_vectors[key] + sigma * (hp_vectors[key] - best_vector);
string red_string = CreateConfigString(hp_names, reduction);
double red_result = Run(recommender, split, red_string, evaluation_measure);
// replace by reduced vector
results.Remove(key);
hp_vectors.Remove(key);
results[red_string] = red_result;
hp_vectors[red_string] = reduction;
}
results[min_key] = best_result;
hp_vectors[min_key] = best_vector;
results[max_key] = worst_result;
hp_vectors[max_key] = worst_vector;
}
keys = new List<string>(results.Keys);
keys.Sort(delegate(string k1, string k2) { return results[k1].CompareTo(results[k2]); });
// set to best hyperparameter values
Recommender.Configure(recommender, keys.First());
return results[keys.First()];
}
/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">a rating predictor</param>
/// <param name="split">a rating dataset split</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
public static Dictionary<string, double> EvaluateOnSplit(RatingPredictor recommender, ISplit<IRatings> split)
{
return EvaluateOnSplit(recommender, split, false);
}
