Пример #1
0
        public override void Evaluate(EvaluationContext context, Model model, Split split)
        {
            split.UpdateFeedbackSlices();
            Initialize(split);

            // if mode is explicit, make sure all item Ids are added to the ItemsMap dic
            if (CandidateItemsMode == CandidateItems.EXPLICIT && model is MmlRecommender)
            {
                foreach (string itemId in _allCandidateItems)
                {
                    ((MmlRecommender)model).ItemsMap.ToInternalID(itemId);
                }
            }

            var candidateUsers = GetCandidateUsers(split);
            var output         = new List <string>();
            int maxCutOff      = CutOffs.Max();

            Parallel.ForEach(candidateUsers, u =>
            {
                var scoredCandidateItems = GetScoredCandidateItems(model, split, u);

                // for this evaluator only max of NumCandidate and CutOff is considered
                var rankedList = scoredCandidateItems.OrderByDescending(i => i.Item2).Take(maxCutOff);

                string line = u.Id + " " + rankedList.Select(r => string.Format("{0}:{1:0.0000}", r.Item1, r.Item2))
                              .Aggregate((a, b) => a + " " + b);

                output.Add(line);
            });

            OutputFile = string.Format("{0}_{1}_{2}.{3}", OutputFile.GetPathWithoutExtension(), split.Id, model.Id, OutputFile.GetFileExtension());
            File.WriteAllLines(OutputFile, output);

            var results = new Dictionary <string, string>();

            results.Add("CandidatesMode", CandidateItemsMode.ToString());
            if (CandidateItemsMode == CandidateItems.EXPLICIT)
            {
                results.Add("CandidatesFile", CandidateItemsFile.Substring(CandidateItemsFile.LastIndexOf('\\') + 1));
            }
            results.Add("NumCandidates", NumCandidates.Max().ToString());
            results.Add("CutOff", maxCutOff.ToString());
            results.Add("OutputFile", OutputFile);
            results.Add("EvalMethod", GetEvaluatorName());

            context.AddResultsSet("rankingMeasures", results);
        }
Пример #2
0
        public override void Evaluate(EvaluationContext context, Model model, Split split)
        {
            split.UpdateFeedbackSlices();
            Initialize(split);

            // if mode is explicit, make sure all item Ids are added to the ItemsMap dic
            if (CandidateItemsMode == CandidateItems.EXPLICIT && model is MmlRecommender)
            {
                foreach (string itemId in _allCandidateItems)
                {
                    ((MmlRecommender)model).ItemsMap.ToInternalID(itemId);
                }
            }

            var testUsers = GetCandidateUsers(split);
            int testedUsersCount = 0, testedCases = 0;
            var precision     = new MultiKeyDictionary <int, int, double>();
            var recall        = new MultiKeyDictionary <int, int, double>();
            var ndcg          = new MultiKeyDictionary <int, int, double>();
            var mrrs          = new MultiKeyDictionary <int, int, double>();
            var maps          = new MultiKeyDictionary <int, int, double>();
            var errs          = new MultiKeyDictionary <int, int, double>();
            var distinctItems = new MultiKeyDictionary <int, int, List <string> >();


            // pre-compute IDCGs for speed up
            var idcgs = new Dictionary <int, double>();

            for (int k = 1; k <= CutOffs.Max(); k++)
            {
                idcgs[k] = Enumerable.Range(1, k).Sum(i => 1.0 / Math.Log(i + 1, 2));
            }

            // initialize measures
            foreach (int maxCand in NumCandidates)
            {
                foreach (int k in CutOffs)
                {
                    precision[maxCand, k]     = 0;
                    recall[maxCand, k]        = 0;
                    ndcg[maxCand, k]          = 0;
                    mrrs[maxCand, k]          = 0;
                    errs[maxCand, k]          = 0;
                    maps[maxCand, k]          = 0;
                    distinctItems[maxCand, k] = new List <string>();

                    if (_perUserMetrics != null)
                    {
                        string path = SetupParameters["userMetricsFile"];
                        _perUserMetrics[maxCand, k] = new StreamWriter(
                            string.Format("{0}_{1}_{2}_{3}_{4}.{5}", path.GetPathWithoutExtension(), split.Id, model.Id, maxCand, k, path.GetFileExtension()));

                        _perUserMetrics[maxCand, k].WriteLine("UserId\tRecall");
                    }
                }
            }

            Parallel.ForEach(testUsers, u =>
            {
                // the followings are heavy processes, the results are stored in lists to prevent over computing
                var scoredRelevantItems  = GetScoredRelevantItems(model, split, u).ToList();
                var scoredCandidateItems = GetScoredCandidateItems(model, split, u).ToList();

                if (scoredRelevantItems.Count == 0)
                {
                    return;
                }

                testedUsersCount++;
                testedCases += scoredRelevantItems.Count;

                // calculating measures for each numCandidates and cutoffs
                foreach (int maxCand in NumCandidates)
                {
                    //var candidatesRankedList = scoredCandidateItems.Take(maxCand).OrderByDescending(i => i.Item2).ToList();
                    var rankedList = scoredCandidateItems.Take(maxCand).Union(scoredRelevantItems).OrderByDescending(i => i.Item2).ToList();

                    foreach (int k in CutOffs)
                    {
                        var topkItems = rankedList.Take(k).Select(ri => ri.Item1).ToList();

                        // calculating diversity of recommendations in terms of number of distinct items
                        distinctItems[maxCand, k] = distinctItems[maxCand, k].Union(topkItems).Distinct().ToList();

                        // Calculate precision and recall with conventional method
                        var hitsAtK = topkItems.Intersect(scoredRelevantItems.Select(i => i.Item1));

                        int hitCount   = 0;
                        double dcg     = 0;
                        int lowestRank = int.MaxValue;
                        double map     = 0;
                        double err     = 0;

                        foreach (string item in hitsAtK)
                        {
                            hitCount++;
                            int rank = topkItems.IndexOf(item);
                            dcg     += 1.0 / Math.Log(rank + 2, 2);
                            map     += (double)hitCount / (rank + 1);
                            err     += 1.0f / (rank + 1);
                            if (rank < lowestRank)
                            {
                                lowestRank = rank;
                            }
                        }

                        int minRelevant = Math.Min(k, scoredRelevantItems.Count);

                        double rc = (double)hitCount / scoredRelevantItems.Count;

                        precision[maxCand, k] += (double)hitCount / k;
                        recall[maxCand, k]    += rc;
                        ndcg[maxCand, k]      += dcg / idcgs[minRelevant];
                        maps[maxCand, k]      += map / minRelevant;
                        // implemented based on https://en.wikipedia.org/wiki/Mean_reciprocal_rank
                        float m = (lowestRank < int.MaxValue) ? 1.0f / (lowestRank + 1) : 0;

                        mrrs[maxCand, k] += m;
                        errs[maxCand, k] += err;

                        if (_perUserMetrics != null)
                        {
                            lock (this)
                                _perUserMetrics[maxCand, k].WriteLine("{0}\t{1:0.0000}", u.Id, m);
                        }
                    }
                }
            });

            // aggregating measures and storing the results
            foreach (int maxCand in NumCandidates)
            {
                foreach (int k in CutOffs)
                {
                    precision[maxCand, k] /= testedUsersCount;
                    recall[maxCand, k]    /= testedUsersCount;
                    maps[maxCand, k]      /= testedUsersCount;
                    ndcg[maxCand, k]      /= testedUsersCount;
                    mrrs[maxCand, k]      /= testedUsersCount;

                    var results = new Dictionary <string, string>();
                    results.Add("TestedCases", testedCases.ToString());
                    results.Add("CandidatesMode", CandidateItemsMode.ToString());
                    if (CandidateItemsMode == CandidateItems.EXPLICIT)
                    {
                        results.Add("CandidatesFile", CandidateItemsFile.Substring(CandidateItemsFile.LastIndexOf('\\') + 1));
                    }
                    results.Add("NumCandidates", maxCand == int.MaxValue ? "max" : maxCand.ToString());
                    results.Add("AllCandidates", _allCandidateItems.Count.ToString());
                    results.Add("CutOff", k.ToString());
                    results.Add("Precision", string.Format("{0:0.0000}", precision[maxCand, k]));
                    results.Add("Recall", string.Format("{0:0.0000}", recall[maxCand, k]));
                    results.Add("MAP", string.Format("{0:0.0000}", maps[maxCand, k]));
                    results.Add("MRR", string.Format("{0:0.0000}", mrrs[maxCand, k]));
                    results.Add("NDCG", string.Format("{0:0.0000}", ndcg[maxCand, k]));
                    results.Add("TotalRecomItems", distinctItems[maxCand, k].Count.ToString());
                    results.Add("%Coverage", string.Format("{0:0.00}",
                                                           (100f * distinctItems[maxCand, k].Count / _allCandidateItems.Count)));
                    results.Add("EvalMethod", GetEvaluatorName());

                    context.AddResultsSet("rankingMeasures", results);

                    if (_perUserMetrics != null)
                    {
                        _perUserMetrics[maxCand, k].Flush();
                        _perUserMetrics[maxCand, k].Close();
                    }
                }
            }
        }
        public override void Evaluate(EvaluationContext context, Model model, Split split)
        {
            split.UpdateFeedbackSlices();
            Initialize(split);

            var testUsers   = GetCandidateUsers(split);
            int testedCases = 0;
            var recallsOpr  = new MultiKeyDictionary <int, int, double>();
            var ndcgOpr     = new MultiKeyDictionary <int, int, double>();
            var mrrOpr      = new MultiKeyDictionary <int, int, double>();

            // initialize measures
            foreach (int maxCand in NumCandidates)
            {
                foreach (int k in CutOffs)
                {
                    recallsOpr[maxCand, k] = 0;
                    ndcgOpr[maxCand, k]    = 0;
                    mrrOpr[maxCand, k]     = 0;

                    if (_predictionWriter != null)
                    {
                        string path = SetupParameters["predictionFile"];
                        _predictionWriter[maxCand, k] = new StreamWriter(
                            string.Format("{0}_{1}_{2}_{3}_{4}.{5}", path.GetPathWithoutExtension(), split.Id, model.Id, maxCand, k, path.GetFileExtension()));

                        _predictionWriter[maxCand, k].WriteLine("UserId,ItemId,Score,IsCorrect");
                    }
                }
            }

            Parallel.ForEach(testUsers, u =>
            {
                // the followings are heavy processes, the results are stored in lists to prevent over computing
                var scoredRelevantItems = GetScoredRelevantItems(model, split, u).ToList();

                if (scoredRelevantItems.Count == 0)
                {
                    return;
                }

                testedCases += scoredRelevantItems.Count;

                // calculating measures for each numCandidates and cutoffs
                foreach (int maxCand in NumCandidates)
                {
                    // Calculate recall with One-plus-random method
                    foreach (Tuple <string, float> item in scoredRelevantItems)
                    {
                        var scoredCandidateItems = GetScoredCandidateItems(model, split, u, item.Item1).ToList();
                        var candidatesRankedList = scoredCandidateItems.Take(maxCand).OrderByDescending(i => i.Item2).ToList();
                        int rank = IndexOfNewItem(candidatesRankedList, item.Item2);
                        foreach (int k in CutOffs)
                        {
                            bool correct = false;

                            if (rank < k)
                            {
                                // if the relevant items falls into the top k items recall would be one (because the only relevent items is covered)
                                // IDCG would be one as well (if relevant item appears in the first position)
                                recallsOpr[maxCand, k] += 1;
                                ndcgOpr[maxCand, k]    += 1.0 / Math.Log(rank + 2, 2);
                                mrrOpr[maxCand, k]     += 1.0 / (rank + 1);
                                correct = true;
                            }

                            if (_predictionWriter != null)
                            {
                                lock (this)
                                    _predictionWriter[maxCand, k].WriteLine("{0},{1},{2},{3}", u.Id, item.Item1, item.Item2, correct ? 1 : 0);
                            }
                        }
                    }
                }
            });

            // aggregating measures and storing the results
            foreach (int maxCand in NumCandidates)
            {
                foreach (int k in CutOffs)
                {
                    recallsOpr[maxCand, k] /= testedCases;
                    ndcgOpr[maxCand, k]    /= testedCases;
                    mrrOpr[maxCand, k]     /= testedCases;

                    var results = new Dictionary <string, string>();
                    results.Add("TestedCases", testedCases.ToString());
                    results.Add("CandidatesMode", CandidateItemsMode.ToString());
                    if (CandidateItemsMode == CandidateItems.EXPLICIT)
                    {
                        results.Add("CandidatesFile", CandidateItemsFile.Substring(CandidateItemsFile.LastIndexOf('\\') + 1));
                    }
                    results.Add("NumCandidates", maxCand == int.MaxValue ? "max" : maxCand.ToString());
                    results.Add("CutOff", k.ToString());
                    results.Add("Recall", string.Format("{0:0.0000}", recallsOpr[maxCand, k]));
                    results.Add("MRR", string.Format("{0:0.0000}", mrrOpr[maxCand, k]));
                    results.Add("NDCG", string.Format("{0:0.0000}", ndcgOpr[maxCand, k]));
                    results.Add("EvalMethod", GetEvaluatorName());

                    context.AddResultsSet("rankingMeasures", results);

                    if (_predictionWriter != null)
                    {
                        _predictionWriter[maxCand, k].Close();
                    }
                }
            }
        }