[Test()] public void TestTranspose()
{
var matrix = new SparseBooleanMatrix();
for (int i = 0; i < 7; i++)
if(i != 2 && i != 4)
{
matrix[i, 1] = true;
matrix[i, 4] = true;
}
matrix[2, 2] = true;
matrix[2, 5] = true;
matrix[4, 3] = true;
// transpose the matrix
var transposed_matrix = (IBooleanMatrix) matrix.Transpose();
// test the transposed matrix
Assert.IsTrue(transposed_matrix[1,0]);
Assert.IsTrue(transposed_matrix[4, 6]);
Assert.IsFalse(transposed_matrix[3, 1]);
Assert.IsFalse(transposed_matrix[5, 4]);
}
/// <summary>Evaluation for rankings of filtered items</summary>
/// <remarks>
/// </remarks>
/// <param name="recommender">item recommender</param>
/// <param name="test">test cases</param>
/// <param name="train">training data</param>
/// <param name="item_attributes">the item attributes to be used for filtering</param>
/// <param name="relevant_users">a collection of integers with all relevant users</param>
/// <param name="relevant_items">a collection of integers with all relevant items</param>
/// <returns>a dictionary containing the evaluation results</returns>
public static Dictionary<string, double> Evaluate(
IItemRecommender recommender,
IPosOnlyFeedback test,
IPosOnlyFeedback train,
SparseBooleanMatrix item_attributes,
ICollection<int> relevant_users,
ICollection<int> relevant_items)
{
if (train.Overlap(test) > 0)
Console.Error.WriteLine("WARNING: Overlapping train and test data");
SparseBooleanMatrix items_by_attribute = (SparseBooleanMatrix) item_attributes.Transpose();
// compute evaluation measures
double auc_sum = 0;
double map_sum = 0;
double prec_5_sum = 0;
double prec_10_sum = 0;
double prec_15_sum = 0;
double ndcg_sum = 0;
// for counting the users and the evaluation lists
int num_lists = 0;
int num_users = 0;
int last_user_id = -1;
foreach (int user_id in relevant_users)
{
var filtered_items = GetFilteredItems(user_id, test, item_attributes);
foreach (int attribute_id in filtered_items.Keys)
{
// TODO optimize this a bit, currently it is quite naive
var relevant_filtered_items = new HashSet<int>(items_by_attribute[attribute_id]);
relevant_filtered_items.IntersectWith(relevant_items);
var correct_items = new HashSet<int>(filtered_items[attribute_id]);
correct_items.IntersectWith(relevant_filtered_items);
// the number of items that are really relevant for this user
var relevant_items_in_train = new HashSet<int>(train.UserMatrix[user_id]);
relevant_items_in_train.IntersectWith(relevant_filtered_items);
int num_eval_items = relevant_filtered_items.Count - relevant_items_in_train.Count();
// skip all users that have 0 or #relevant_filtered_items test items
if (correct_items.Count == 0)
continue;
if (num_eval_items - correct_items.Count == 0)
continue;
// counting stats
num_lists++;
if (last_user_id != user_id)
{
last_user_id = user_id;
num_users++;
}
// evaluation
int[] prediction = Prediction.PredictItems(recommender, user_id, relevant_filtered_items);
auc_sum += Items.AUC(prediction, correct_items, train.UserMatrix[user_id]);
map_sum += Items.MAP(prediction, correct_items, train.UserMatrix[user_id]);
ndcg_sum += Items.NDCG(prediction, correct_items, train.UserMatrix[user_id]);
prec_5_sum += Items.PrecisionAt(prediction, correct_items, train.UserMatrix[user_id], 5);
prec_10_sum += Items.PrecisionAt(prediction, correct_items, train.UserMatrix[user_id], 10);
prec_15_sum += Items.PrecisionAt(prediction, correct_items, train.UserMatrix[user_id], 15);
if (prediction.Length != relevant_filtered_items.Count)
throw new Exception("Not all items have been ranked.");
if (num_lists % 1000 == 0)
Console.Error.Write(".");
if (num_lists % 20000 == 0)
Console.Error.WriteLine();
}
}
var result = new Dictionary<string, double>();
result.Add("AUC", auc_sum / num_lists);
result.Add("MAP", map_sum / num_lists);
result.Add("NDCG", ndcg_sum / num_lists);
result.Add("prec@5", prec_5_sum / num_lists);
result.Add("prec@10", prec_10_sum / num_lists);
result.Add("prec@15", prec_15_sum / num_lists);
result.Add("num_users", num_users);
result.Add("num_lists", num_lists);
result.Add("num_items", relevant_items.Count);
return result;
}
