public AbsoluteEffectivenessEstimate Estimate(Run run, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
{
double e = 0, var = 0;
// Traverse docs retrieved
foreach (string doc in run.Documents) {
RelevanceEstimate docEst = relEstimator.Estimate(run.Query, doc);
e += docEst.Expectation;
var += docEst.Variance;
}
// Compute average
e /= run.Documents.Count();
var /= run.Documents.Count() * run.Documents.Count();
// Normalize between 0 and 1
e /= this.MaxRelevance;
var /= this.MaxRelevance * this.MaxRelevance;
Estimate est = new Estimate(e, var);
return new AbsoluteEffectivenessEstimate(run.System, run.Query,
e, var,
confEstimator.EstimateInterval(est), confEstimator.EstimateAbsoluteConfidence(est));
}
public RelativeEffectivenessEstimate Estimate(Run runA, Run runB, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
{
double e = 0, var = 0;
// Traverse docs retrieved by A
HashSet<string> inRunA = new HashSet<string>(); // retrieved by run A
foreach (string doc in runA.Documents) {
RelevanceEstimate docEst = relEstimator.Estimate(runA.Query, doc);
e += docEst.Expectation;
var += docEst.Variance;
inRunA.Add(doc);
}
// Traverse docs retrieved by B
foreach (string doc in runB.Documents) {
RelevanceEstimate docEst = relEstimator.Estimate(runB.Query, doc);
e -= docEst.Expectation;
if (inRunA.Contains(doc)) {
// If retrieved in both runs, does not contribute to variance
var -= docEst.Variance;
} else {
var += docEst.Variance;
}
}
// Compute average
e /= inRunA.Count;
var /= inRunA.Count * inRunA.Count;
// Normalize between 0 and 1
e /= this.MaxRelevance;
var /= this.MaxRelevance * this.MaxRelevance;
Estimate est = new Estimate(e, var);
return new RelativeEffectivenessEstimate(runA.System, runB.System, runA.Query,
e, var,
confEstimator.EstimateInterval(est), confEstimator.EstimateRelativeConfidence(est));
}