internal static List<AbsoluteEffectivenessEstimate> GetSortedMeanAbsolutes(
Dictionary<string, Dictionary<string, AbsoluteEffectivenessEstimate>> sqAbss,
IConfidenceEstimator confEstimator)
{
// Compute means
List<AbsoluteEffectivenessEstimate> absSorted = new List<AbsoluteEffectivenessEstimate>();
foreach (var sqAbsEst in sqAbss) {
double e = sqAbsEst.Value.Sum(qAbsEst => qAbsEst.Value.Expectation);
double var = sqAbsEst.Value.Sum(qAbsEst => qAbsEst.Value.Variance);
e /= sqAbsEst.Value.Count;
var /= sqAbsEst.Value.Count * sqAbsEst.Value.Count;
Estimate est = new Estimate(e, var);
absSorted.Add(new AbsoluteEffectivenessEstimate(sqAbsEst.Key, "[all]",
e, var,
confEstimator.EstimateInterval(est), confEstimator.EstimateAbsoluteConfidence(est)));
}
// and sort
absSorted = absSorted.OrderByDescending(est => est.Expectation).ToList();
return absSorted;
}
internal static List<RelativeEffectivenessEstimate> GetSortedMeanRelatives(
Dictionary<string, Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>>> ssqRels,
IConfidenceEstimator confEstimator)
{
// Compute means
List<RelativeEffectivenessEstimate> rels = new List<RelativeEffectivenessEstimate>();
foreach (var sqRels in ssqRels) {
foreach (var qRels in sqRels.Value) {
string sysA = sqRels.Key;
string sysB = qRels.Key;
double e = qRels.Value.Values.Sum(relEst => relEst.Expectation);
double var = qRels.Value.Values.Sum(relEst => relEst.Variance);
e /= qRels.Value.Values.Count;
var /= qRels.Value.Values.Count * qRels.Value.Values.Count;
if (e < 0) {
e = -e;
sysA = qRels.Key;
sysB = sqRels.Key;
}
Estimate est = new Estimate(e, var);
rels.Add(new RelativeEffectivenessEstimate(sysA, sysB, "[all]",
e, var,
confEstimator.EstimateInterval(est), confEstimator.EstimateRelativeConfidence(est)));
}
}
// and sort
var groups = rels.GroupBy(r => r.SystemA).OrderByDescending(g => g.Count());
List<RelativeEffectivenessEstimate> relSorted = new List<RelativeEffectivenessEstimate>();
foreach (var group in groups) {
relSorted.AddRange(group.OrderBy(r => r.Expectation));
}
return relSorted;
}
public double EstimateRelativeConfidence(Estimate e)
{
return NormalConfidenceEstimator.CDF((e.Expectation-this._sizeRel) / Math.Sqrt(e.Variance));
}
public double[] EstimateInterval(Estimate e)
{
double z = NormalConfidenceEstimator.Quantile((1.0 - this._confidence) / 2.0);
double len = Math.Abs(z * Math.Sqrt(e.Variance));
return new double[] { e.Expectation - len, e.Expectation + len };
}
public double EstimateAbsoluteConfidence(Estimate e)
{
return 1.0 - 2 * NormalConfidenceEstimator.CDF(-this._sizeAbs / Math.Sqrt(e.Variance));
}
public AbsoluteEffectivenessEstimate Estimate(Run run, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
{
double e = 0, var = 0;
// Traverse docs retrieved
foreach (string doc in run.Documents) {
RelevanceEstimate docEst = relEstimator.Estimate(run.Query, doc);
e += docEst.Expectation;
var += docEst.Variance;
}
// Compute average
e /= run.Documents.Count();
var /= run.Documents.Count() * run.Documents.Count();
// Normalize between 0 and 1
e /= this.MaxRelevance;
var /= this.MaxRelevance * this.MaxRelevance;
Estimate est = new Estimate(e, var);
return new AbsoluteEffectivenessEstimate(run.System, run.Query,
e, var,
confEstimator.EstimateInterval(est), confEstimator.EstimateAbsoluteConfidence(est));
}
public RelativeEffectivenessEstimate Estimate(Run runA, Run runB, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
{
double e = 0, var = 0;
// Traverse docs retrieved by A
HashSet<string> inRunA = new HashSet<string>(); // retrieved by run A
foreach (string doc in runA.Documents) {
RelevanceEstimate docEst = relEstimator.Estimate(runA.Query, doc);
e += docEst.Expectation;
var += docEst.Variance;
inRunA.Add(doc);
}
// Traverse docs retrieved by B
foreach (string doc in runB.Documents) {
RelevanceEstimate docEst = relEstimator.Estimate(runB.Query, doc);
e -= docEst.Expectation;
if (inRunA.Contains(doc)) {
// If retrieved in both runs, does not contribute to variance
var -= docEst.Variance;
} else {
var += docEst.Variance;
}
}
// Compute average
e /= inRunA.Count;
var /= inRunA.Count * inRunA.Count;
// Normalize between 0 and 1
e /= this.MaxRelevance;
var /= this.MaxRelevance * this.MaxRelevance;
Estimate est = new Estimate(e, var);
return new RelativeEffectivenessEstimate(runA.System, runB.System, runA.Query,
e, var,
confEstimator.EstimateInterval(est), confEstimator.EstimateRelativeConfidence(est));
}
