public override void Evaluate(EvaluationContext context, Model model, Split split) { split.UpdateFeedbackSlices(); Initialize(split); // if mode is explicit, make sure all item Ids are added to the ItemsMap dic if (CandidateItemsMode == CandidateItems.EXPLICIT && model is MmlRecommender) { foreach (string itemId in _allCandidateItems) { ((MmlRecommender)model).ItemsMap.ToInternalID(itemId); } } var candidateUsers = GetCandidateUsers(split); var output = new List <string>(); int maxCutOff = CutOffs.Max(); Parallel.ForEach(candidateUsers, u => { var scoredCandidateItems = GetScoredCandidateItems(model, split, u); // for this evaluator only max of NumCandidate and CutOff is considered var rankedList = scoredCandidateItems.OrderByDescending(i => i.Item2).Take(maxCutOff); string line = u.Id + " " + rankedList.Select(r => string.Format("{0}:{1:0.0000}", r.Item1, r.Item2)) .Aggregate((a, b) => a + " " + b); output.Add(line); }); OutputFile = string.Format("{0}_{1}_{2}.{3}", OutputFile.GetPathWithoutExtension(), split.Id, model.Id, OutputFile.GetFileExtension()); File.WriteAllLines(OutputFile, output); var results = new Dictionary <string, string>(); results.Add("CandidatesMode", CandidateItemsMode.ToString()); if (CandidateItemsMode == CandidateItems.EXPLICIT) { results.Add("CandidatesFile", CandidateItemsFile.Substring(CandidateItemsFile.LastIndexOf('\\') + 1)); } results.Add("NumCandidates", NumCandidates.Max().ToString()); results.Add("CutOff", maxCutOff.ToString()); results.Add("OutputFile", OutputFile); results.Add("EvalMethod", GetEvaluatorName()); context.AddResultsSet("rankingMeasures", results); }
public override void Setup() { // candidate items if (!SetupParameters.ContainsKey("candidateItemsMode")) { CandidateItemsMode = CandidateItems.TRAINING; } else { CandidateItemsMode = (CandidateItems)Enum.Parse(typeof(CandidateItems), SetupParameters["candidateItemsMode"], true); } if (SetupParameters.ContainsKey("candidateItemsFile")) { CandidateItemsFile = SetupParameters["candidateItemsFile"]; } else if (CandidateItemsMode == CandidateItems.EXPLICIT) { throw new WrapRecException("Expect a 'candidateItemsFile' for the mode 'explicit'!"); } // candidate users if (!SetupParameters.ContainsKey("candidateUsersMode")) { CandidateUsersMode = CandidateItems.TEST; } else { CandidateUsersMode = (CandidateItems)Enum.Parse(typeof(CandidateItems), SetupParameters["candidateUsersMode"], true); } if (SetupParameters.ContainsKey("candidateUsersFile")) { CandidateUsersFile = SetupParameters["candidateUsersFile"]; } else if (CandidateUsersMode == CandidateItems.EXPLICIT) { throw new WrapRecException("Expect a 'candidateUsersFile' for the mode 'explicit!'"); } CutOffs = SetupParameters["cutOffs"].Split(',').Select(c => int.Parse(c)).ToArray(); if (!SetupParameters.ContainsKey("numCandidates")) { NumCandidates = new int[] { int.MaxValue } } ; else { NumCandidates = SetupParameters["numCandidates"].Split(',').Select(n => { return((n == "max") ? int.MaxValue : int.Parse(n)); }).ToArray(); } _maxNumCandidates = NumCandidates.Max(); if (SetupParameters.ContainsKey("relevantItems") && SetupParameters["relevantItems"].ToLower() == "all") { _isSliceRelevant = f => f.SliceType == FeedbackSlice.TRAIN || f.SliceType == FeedbackSlice.TEST; } else { _isSliceRelevant = f => f.SliceType == FeedbackSlice.TEST; } if (SetupParameters.ContainsKey("userMetricsFile")) { _perUserMetrics = new MultiKeyDictionary <int, int, StreamWriter>(); } }