public override void Evaluate(EvaluationContext context, Model model, Split split)
{
split.UpdateFeedbackSlices();
Initialize(split);
// if mode is explicit, make sure all item Ids are added to the ItemsMap dic
if (CandidateItemsMode == CandidateItems.EXPLICIT && model is MmlRecommender)
{
foreach (string itemId in _allCandidateItems)
{
((MmlRecommender)model).ItemsMap.ToInternalID(itemId);
}
}
var candidateUsers = GetCandidateUsers(split);
var output = new List <string>();
int maxCutOff = CutOffs.Max();
Parallel.ForEach(candidateUsers, u =>
{
var scoredCandidateItems = GetScoredCandidateItems(model, split, u);
// for this evaluator only max of NumCandidate and CutOff is considered
var rankedList = scoredCandidateItems.OrderByDescending(i => i.Item2).Take(maxCutOff);
string line = u.Id + " " + rankedList.Select(r => string.Format("{0}:{1:0.0000}", r.Item1, r.Item2))
.Aggregate((a, b) => a + " " + b);
output.Add(line);
});
OutputFile = string.Format("{0}_{1}_{2}.{3}", OutputFile.GetPathWithoutExtension(), split.Id, model.Id, OutputFile.GetFileExtension());
File.WriteAllLines(OutputFile, output);
var results = new Dictionary <string, string>();
results.Add("CandidatesMode", CandidateItemsMode.ToString());
if (CandidateItemsMode == CandidateItems.EXPLICIT)
{
results.Add("CandidatesFile", CandidateItemsFile.Substring(CandidateItemsFile.LastIndexOf('\\') + 1));
}
results.Add("NumCandidates", NumCandidates.Max().ToString());
results.Add("CutOff", maxCutOff.ToString());
results.Add("OutputFile", OutputFile);
results.Add("EvalMethod", GetEvaluatorName());
context.AddResultsSet("rankingMeasures", results);
}
public override void Setup()
{
// candidate items
if (!SetupParameters.ContainsKey("candidateItemsMode"))
{
CandidateItemsMode = CandidateItems.TRAINING;
}
else
{
CandidateItemsMode = (CandidateItems)Enum.Parse(typeof(CandidateItems), SetupParameters["candidateItemsMode"], true);
}
if (SetupParameters.ContainsKey("candidateItemsFile"))
{
CandidateItemsFile = SetupParameters["candidateItemsFile"];
}
else if (CandidateItemsMode == CandidateItems.EXPLICIT)
{
throw new WrapRecException("Expect a 'candidateItemsFile' for the mode 'explicit'!");
}
// candidate users
if (!SetupParameters.ContainsKey("candidateUsersMode"))
{
CandidateUsersMode = CandidateItems.TEST;
}
else
{
CandidateUsersMode = (CandidateItems)Enum.Parse(typeof(CandidateItems), SetupParameters["candidateUsersMode"], true);
}
if (SetupParameters.ContainsKey("candidateUsersFile"))
{
CandidateUsersFile = SetupParameters["candidateUsersFile"];
}
else if (CandidateUsersMode == CandidateItems.EXPLICIT)
{
throw new WrapRecException("Expect a 'candidateUsersFile' for the mode 'explicit!'");
}
CutOffs = SetupParameters["cutOffs"].Split(',').Select(c => int.Parse(c)).ToArray();
if (!SetupParameters.ContainsKey("numCandidates"))
{
NumCandidates = new int[] { int.MaxValue }
}
;
else
{
NumCandidates = SetupParameters["numCandidates"].Split(',').Select(n =>
{
return((n == "max") ? int.MaxValue : int.Parse(n));
}).ToArray();
}
_maxNumCandidates = NumCandidates.Max();
if (SetupParameters.ContainsKey("relevantItems") && SetupParameters["relevantItems"].ToLower() == "all")
{
_isSliceRelevant = f => f.SliceType == FeedbackSlice.TRAIN || f.SliceType == FeedbackSlice.TEST;
}
else
{
_isSliceRelevant = f => f.SliceType == FeedbackSlice.TEST;
}
if (SetupParameters.ContainsKey("userMetricsFile"))
{
_perUserMetrics = new MultiKeyDictionary <int, int, StreamWriter>();
}
}