internal static List<AbsoluteEffectivenessEstimate> GetSortedMeanAbsolutes( Dictionary<string, Dictionary<string, AbsoluteEffectivenessEstimate>> sqAbss, IConfidenceEstimator confEstimator) { // Compute means List<AbsoluteEffectivenessEstimate> absSorted = new List<AbsoluteEffectivenessEstimate>(); foreach (var sqAbsEst in sqAbss) { double e = sqAbsEst.Value.Sum(qAbsEst => qAbsEst.Value.Expectation); double var = sqAbsEst.Value.Sum(qAbsEst => qAbsEst.Value.Variance); e /= sqAbsEst.Value.Count; var /= sqAbsEst.Value.Count * sqAbsEst.Value.Count; Estimate est = new Estimate(e, var); absSorted.Add(new AbsoluteEffectivenessEstimate(sqAbsEst.Key, "[all]", e, var, confEstimator.EstimateInterval(est), confEstimator.EstimateAbsoluteConfidence(est))); } // and sort absSorted = absSorted.OrderByDescending(est => est.Expectation).ToList(); return absSorted; }
internal static List<RelativeEffectivenessEstimate> GetSortedMeanRelatives( Dictionary<string, Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>>> ssqRels, IConfidenceEstimator confEstimator) { // Compute means List<RelativeEffectivenessEstimate> rels = new List<RelativeEffectivenessEstimate>(); foreach (var sqRels in ssqRels) { foreach (var qRels in sqRels.Value) { string sysA = sqRels.Key; string sysB = qRels.Key; double e = qRels.Value.Values.Sum(relEst => relEst.Expectation); double var = qRels.Value.Values.Sum(relEst => relEst.Variance); e /= qRels.Value.Values.Count; var /= qRels.Value.Values.Count * qRels.Value.Values.Count; if (e < 0) { e = -e; sysA = qRels.Key; sysB = sqRels.Key; } Estimate est = new Estimate(e, var); rels.Add(new RelativeEffectivenessEstimate(sysA, sysB, "[all]", e, var, confEstimator.EstimateInterval(est), confEstimator.EstimateRelativeConfidence(est))); } } // and sort var groups = rels.GroupBy(r => r.SystemA).OrderByDescending(g => g.Count()); List<RelativeEffectivenessEstimate> relSorted = new List<RelativeEffectivenessEstimate>(); foreach (var group in groups) { relSorted.AddRange(group.OrderBy(r => r.Expectation)); } return relSorted; }
public double EstimateRelativeConfidence(Estimate e) { return NormalConfidenceEstimator.CDF((e.Expectation-this._sizeRel) / Math.Sqrt(e.Variance)); }
public double[] EstimateInterval(Estimate e) { double z = NormalConfidenceEstimator.Quantile((1.0 - this._confidence) / 2.0); double len = Math.Abs(z * Math.Sqrt(e.Variance)); return new double[] { e.Expectation - len, e.Expectation + len }; }
public double EstimateAbsoluteConfidence(Estimate e) { return 1.0 - 2 * NormalConfidenceEstimator.CDF(-this._sizeAbs / Math.Sqrt(e.Variance)); }
public AbsoluteEffectivenessEstimate Estimate(Run run, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator) { double e = 0, var = 0; // Traverse docs retrieved foreach (string doc in run.Documents) { RelevanceEstimate docEst = relEstimator.Estimate(run.Query, doc); e += docEst.Expectation; var += docEst.Variance; } // Compute average e /= run.Documents.Count(); var /= run.Documents.Count() * run.Documents.Count(); // Normalize between 0 and 1 e /= this.MaxRelevance; var /= this.MaxRelevance * this.MaxRelevance; Estimate est = new Estimate(e, var); return new AbsoluteEffectivenessEstimate(run.System, run.Query, e, var, confEstimator.EstimateInterval(est), confEstimator.EstimateAbsoluteConfidence(est)); }
public RelativeEffectivenessEstimate Estimate(Run runA, Run runB, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator) { double e = 0, var = 0; // Traverse docs retrieved by A HashSet<string> inRunA = new HashSet<string>(); // retrieved by run A foreach (string doc in runA.Documents) { RelevanceEstimate docEst = relEstimator.Estimate(runA.Query, doc); e += docEst.Expectation; var += docEst.Variance; inRunA.Add(doc); } // Traverse docs retrieved by B foreach (string doc in runB.Documents) { RelevanceEstimate docEst = relEstimator.Estimate(runB.Query, doc); e -= docEst.Expectation; if (inRunA.Contains(doc)) { // If retrieved in both runs, does not contribute to variance var -= docEst.Variance; } else { var += docEst.Variance; } } // Compute average e /= inRunA.Count; var /= inRunA.Count * inRunA.Count; // Normalize between 0 and 1 e /= this.MaxRelevance; var /= this.MaxRelevance * this.MaxRelevance; Estimate est = new Estimate(e, var); return new RelativeEffectivenessEstimate(runA.System, runB.System, runA.Query, e, var, confEstimator.EstimateInterval(est), confEstimator.EstimateRelativeConfidence(est)); }