internal static List<RelativeEffectivenessEstimate> GetSortedMeanRelatives(
Dictionary<string, Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>>> ssqRels,
IConfidenceEstimator confEstimator)
{
// Compute means
List<RelativeEffectivenessEstimate> rels = new List<RelativeEffectivenessEstimate>();
foreach (var sqRels in ssqRels) {
foreach (var qRels in sqRels.Value) {
string sysA = sqRels.Key;
string sysB = qRels.Key;
double e = qRels.Value.Values.Sum(relEst => relEst.Expectation);
double var = qRels.Value.Values.Sum(relEst => relEst.Variance);
e /= qRels.Value.Values.Count;
var /= qRels.Value.Values.Count * qRels.Value.Values.Count;
if (e < 0) {
e = -e;
sysA = qRels.Key;
sysB = sqRels.Key;
}
Estimate est = new Estimate(e, var);
rels.Add(new RelativeEffectivenessEstimate(sysA, sysB, "[all]",
e, var,
confEstimator.EstimateInterval(est), confEstimator.EstimateRelativeConfidence(est)));
}
}
// and sort
var groups = rels.GroupBy(r => r.SystemA).OrderByDescending(g => g.Count());
List<RelativeEffectivenessEstimate> relSorted = new List<RelativeEffectivenessEstimate>();
foreach (var group in groups) {
relSorted.AddRange(group.OrderBy(r => r.Expectation));
}
return relSorted;
}
public RelativeEffectivenessEstimate Estimate(Run runA, Run runB, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
{
double e = 0, var = 0;
// Traverse docs retrieved by A
HashSet<string> inRunA = new HashSet<string>(); // retrieved by run A
foreach (string doc in runA.Documents) {
RelevanceEstimate docEst = relEstimator.Estimate(runA.Query, doc);
e += docEst.Expectation;
var += docEst.Variance;
inRunA.Add(doc);
}
// Traverse docs retrieved by B
foreach (string doc in runB.Documents) {
RelevanceEstimate docEst = relEstimator.Estimate(runB.Query, doc);
e -= docEst.Expectation;
if (inRunA.Contains(doc)) {
// If retrieved in both runs, does not contribute to variance
var -= docEst.Variance;
} else {
var += docEst.Variance;
}
}
// Compute average
e /= inRunA.Count;
var /= inRunA.Count * inRunA.Count;
// Normalize between 0 and 1
e /= this.MaxRelevance;
var /= this.MaxRelevance * this.MaxRelevance;
Estimate est = new Estimate(e, var);
return new RelativeEffectivenessEstimate(runA.System, runB.System, runA.Query,
e, var,
confEstimator.EstimateInterval(est), confEstimator.EstimateRelativeConfidence(est));
}
internal static List <RelativeEffectivenessEstimate> GetSortedMeanRelatives(
Dictionary <string, Dictionary <string, Dictionary <string, RelativeEffectivenessEstimate> > > ssqRels,
IConfidenceEstimator confEstimator)
{
// Compute means
List <RelativeEffectivenessEstimate> rels = new List <RelativeEffectivenessEstimate>();
foreach (var sqRels in ssqRels)
{
foreach (var qRels in sqRels.Value)
{
string sysA = sqRels.Key;
string sysB = qRels.Key;
double e = qRels.Value.Values.Sum(relEst => relEst.Expectation);
double var = qRels.Value.Values.Sum(relEst => relEst.Variance);
e /= qRels.Value.Values.Count;
var /= qRels.Value.Values.Count * qRels.Value.Values.Count;
if (e < 0)
{
e = -e;
sysA = qRels.Key;
sysB = sqRels.Key;
}
Estimate est = new Estimate(e, var);
rels.Add(new RelativeEffectivenessEstimate(sysA, sysB, "[all]",
e, var,
confEstimator.EstimateInterval(est), confEstimator.EstimateRelativeConfidence(est)));
}
}
// and sort
var groups = rels.GroupBy(r => r.SystemA).OrderByDescending(g => g.Count());
List <RelativeEffectivenessEstimate> relSorted = new List <RelativeEffectivenessEstimate>();
foreach (var group in groups)
{
relSorted.AddRange(group.OrderBy(r => r.Expectation));
}
return(relSorted);
}
public RelativeEffectivenessEstimate Estimate(Run runA, Run runB, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
{
double e = 0, var = 0;
// Traverse docs retrieved by A
HashSet <string> inRunA = new HashSet <string>(); // retrieved by run A
foreach (string doc in runA.Documents)
{
RelevanceEstimate docEst = relEstimator.Estimate(runA.Query, doc);
e += docEst.Expectation;
var += docEst.Variance;
inRunA.Add(doc);
}
// Traverse docs retrieved by B
foreach (string doc in runB.Documents)
{
RelevanceEstimate docEst = relEstimator.Estimate(runB.Query, doc);
e -= docEst.Expectation;
if (inRunA.Contains(doc))
{
// If retrieved in both runs, does not contribute to variance
var -= docEst.Variance;
}
else
{
var += docEst.Variance;
}
}
// Compute average
e /= inRunA.Count;
var /= inRunA.Count * inRunA.Count;
// Normalize between 0 and 1
e /= this.MaxRelevance;
var /= this.MaxRelevance * this.MaxRelevance;
Estimate est = new Estimate(e, var);
return(new RelativeEffectivenessEstimate(runA.System, runB.System, runA.Query,
e, var,
confEstimator.EstimateInterval(est), confEstimator.EstimateRelativeConfidence(est)));
}