public void Initialize(IEnumerable <Run> runs, IEnumerable <RelevanceEstimate> judged)
{
// Re-structure known judgments
foreach (var j in judged)
{
string id = RelevanceEstimate.GetId(j.Query, j.Document);
this._judged[id] = j;
}
// Instantiate estimator
switch (this._name)
{
case "uniform":
// nothing to initialize
this._estimator = new UniformRelevanceEstimator(100);
break;
case "mout":
// read metadata
IEnumerable <Metadata> metadata = AbstractCommand.ReadMetadata(this._parameters["meta"]);
this._estimator = new MoutRelevanceEstimator(runs, metadata);
break;
case "mjud":
// read metadata
metadata = AbstractCommand.ReadMetadata(this._parameters["meta"]);
IEnumerable <RelevanceEstimate> judgedEst = this._parameters.ContainsKey("judged") ?
AbstractCommand.ReadKnownJudgments(this._parameters["judged"]) :
new RelevanceEstimate[] { };
this._estimator = new MjudRelevanceEstimator(runs, metadata, judgedEst);
break;
}
}
public MjudRelevanceEstimator(IEnumerable <Run> runs, IEnumerable <Metadata> metadata, IEnumerable <RelevanceEstimate> judged)
{
// Instantiate model: fSYS, aSYS, aART
this._model = new OrdinalLogisticRegression(MjudRelevanceEstimator.LABELS, MjudRelevanceEstimator.ALPHAS, MjudRelevanceEstimator.BETAS);
this._defaultEstimator = new MoutRelevanceEstimator(runs, metadata);
// Number of systems and metadata
int nSys = runs.Select(r => r.System).Distinct().Count();
this._dArtists = new Dictionary <string, string>();
foreach (var m in metadata)
{
this._dArtists[m.Document] = m.Artist;
}
// fSYS
this._fSYS = new Dictionary <string, double>();
this._aSYS = new Dictionary <string, double>();
this._aART = new Dictionary <string, double>();
this._sRels = new Dictionary <string, List <double> >();
this._qaRels = new Dictionary <string, List <double> >();
foreach (var run in runs)
{
string query = run.Query;
foreach (string doc in run.Documents)
{
string id = RelevanceEstimate.GetId(query, doc);
// fSYS
double fSYS = 0;
this._fSYS.TryGetValue(id, out fSYS);
this._fSYS[id] = fSYS + 1.0 / nSys;
this._aSYS[id] = 0;
this._aART[id] = 0;
// sRels
if (!this._sRels.ContainsKey(run.System))
{
this._sRels[run.System] = new List <double>();
}
// qaRels
string artist = null;
if (this._dArtists.TryGetValue(doc, out artist) && !this._qaRels.ContainsKey(query + "\t" + artist))
{
this._qaRels[query + "\t" + artist] = new List <double>();
}
}
}
// OV
this._OV = ((double)this._fSYS.Count) / (nSys * (runs.Count() / nSys) * runs.First().Documents.Count());
this._qdsRanks = jurbano.Allcea.Cli.AbstractCommand.ToQueryDocumentSystemRanks(runs);
// Incorporate known judgments
foreach (var est in judged)
{
this.Update(est);
}
this._needsUpdate = true;
}
public EstimatorWrapper(string name, Dictionary <string, string> parameters)
{
this._judged = new Dictionary <string, RelevanceEstimate>();
this._name = name;
this._estimator = null;
this._parameters = parameters;
switch (this._name)
{
case "uniform":
if (parameters.Count != 0)
{
throw new ParseException("Estimator 'uniform' does not have parameters.");
}
break;
case "mout":
if (parameters.Count != 1 || !parameters.ContainsKey("meta"))
{
throw new ParseException("Invalid parameters for estimator 'mout'.");
}
string metadataPath = parameters["meta"];
if (!File.Exists(metadataPath))
{
throw new ArgumentException("Metadata file '" + metadataPath + "' does not exist.");
}
break;
case "mjud":
if (!((parameters.Count == 1 && parameters.ContainsKey("meta")) ||
(parameters.Count == 2 && parameters.ContainsKey("meta") && parameters.ContainsKey("judged"))))
{
throw new ParseException("Invalid parameters for estimator 'mjud'.");
}
metadataPath = parameters["meta"];
if (!File.Exists(metadataPath))
{
throw new ArgumentException("Metadata file '" + metadataPath + "' does not exist.");
}
if (parameters.Count == 2)
{
string judgedPath = parameters["judged"];
if (!File.Exists(judgedPath))
{
throw new ArgumentException("Known judgments file '" + judgedPath + "' does not exist.");
}
}
break;
default:
throw new ParseException("'" + name + "' is not a valid estimator name.");
}
}
protected Dictionary<string, List<double>> _sRels; // [sys, [rel]]
#endregion Fields
#region Constructors
public MjudRelevanceEstimator(IEnumerable<Run> runs, IEnumerable<Metadata> metadata, IEnumerable<RelevanceEstimate> judged)
{
// Instantiate model: fSYS, aSYS, aART
this._model = new OrdinalLogisticRegression(MjudRelevanceEstimator.LABELS, MjudRelevanceEstimator.ALPHAS, MjudRelevanceEstimator.BETAS);
this._defaultEstimator = new MoutRelevanceEstimator(runs, metadata);
// Number of systems and metadata
int nSys = runs.Select(r => r.System).Distinct().Count();
this._dArtists = new Dictionary<string, string>();
foreach (var m in metadata) {
this._dArtists[m.Document] = m.Artist;
}
// fSYS
this._fSYS = new Dictionary<string, double>();
this._aSYS = new Dictionary<string, double>();
this._aART = new Dictionary<string, double>();
this._sRels = new Dictionary<string, List<double>>();
this._qaRels = new Dictionary<string, List<double>>();
foreach (var run in runs) {
string query = run.Query;
foreach (string doc in run.Documents) {
string id = RelevanceEstimate.GetId(query, doc);
// fSYS
double fSYS = 0;
this._fSYS.TryGetValue(id, out fSYS);
this._fSYS[id] = fSYS + 1.0 / nSys;
this._aSYS[id] = 0;
this._aART[id] = 0;
// sRels
if (!this._sRels.ContainsKey(run.System)) {
this._sRels[run.System] = new List<double>();
}
// qaRels
string artist = null;
if (this._dArtists.TryGetValue(doc, out artist) && !this._qaRels.ContainsKey(query + "\t" + artist)) {
this._qaRels[query + "\t" + artist] = new List<double>();
}
}
}
// OV
this._OV = ((double)this._fSYS.Count) / (nSys * (runs.Count()/nSys) * runs.First().Documents.Count());
this._qdsRanks = jurbano.Allcea.Cli.AbstractCommand.ToQueryDocumentSystemRanks(runs);
// Incorporate known judgments
foreach (var est in judged) {
this.Update(est);
}
this._needsUpdate = true;
}
internal static Dictionary <string, Dictionary <string, AbsoluteEffectivenessEstimate> > GetSystemQueryAbsolutes(
Dictionary <string, Dictionary <string, Run> > sqRuns,
IMeasure measure, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
{
Dictionary <string, Dictionary <string, AbsoluteEffectivenessEstimate> > sqAbss = new Dictionary <string, Dictionary <string, AbsoluteEffectivenessEstimate> >();
foreach (var sqRun in sqRuns)
{
Dictionary <string, AbsoluteEffectivenessEstimate> qAbs = new Dictionary <string, AbsoluteEffectivenessEstimate>();
foreach (var qRun in sqRun.Value)
{
qAbs.Add(qRun.Key, measure.Estimate(qRun.Value, relEstimator, confEstimator));
}
sqAbss.Add(sqRun.Key, qAbs);
}
return(sqAbss);
}
internal static Dictionary <string, Dictionary <string, Dictionary <string, RelativeEffectivenessEstimate> > > GetSystemSystemQueryRelatives(
Dictionary <string, Dictionary <string, Run> > sqRuns,
IMeasure measure, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
{
Dictionary <string, Dictionary <string, Dictionary <string, RelativeEffectivenessEstimate> > > ssqRelEstimates =
new Dictionary <string, Dictionary <string, Dictionary <string, RelativeEffectivenessEstimate> > >(); // [sysA [sysB [query rel]]]
string[] allSystems = sqRuns.Keys.ToArray();
Parallel.For(0, allSystems.Length - 1, i => {
string sysA = allSystems[i];
var runsA = sqRuns[sysA];
Dictionary <string, Dictionary <string, RelativeEffectivenessEstimate> > sqRelEstimates = new Dictionary <string, Dictionary <string, RelativeEffectivenessEstimate> >();
for (int j = i + 1; j < allSystems.Length; j++)
{
Dictionary <string, RelativeEffectivenessEstimate> qRelEstimates = new Dictionary <string, RelativeEffectivenessEstimate>();
string sysB = allSystems[j];
var runsB = sqRuns[sysB];
foreach (var qRun in runsA)
{
qRelEstimates.Add(qRun.Key, measure.Estimate(qRun.Value, runsB[qRun.Key], relEstimator, confEstimator));
}
sqRelEstimates.Add(sysB, qRelEstimates);
}
lock (ssqRelEstimates) {
ssqRelEstimates.Add(sysA, sqRelEstimates);
}
});
//for (int i = 0; i < allSystems.Length - 1; i++) {
// string sysA = allSystems[i];
// var runsA = sqRuns[sysA];
// Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>> sqRelEstimates = new Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>>();
// for (int j = i + 1; j < allSystems.Length; j++) {
// Dictionary<string, RelativeEffectivenessEstimate> qRelEstimates = new Dictionary<string, RelativeEffectivenessEstimate>();
// string sysB = allSystems[j];
// var runsB = sqRuns[sysB];
// foreach (var qRun in runsA) {
// qRelEstimates.Add(qRun.Key, measure.Estimate(qRun.Value, runsB[qRun.Key], relEstimator, confEstimator));
// }
// sqRelEstimates.Add(sysB, qRelEstimates);
// }
// ssqRelEstimates.Add(sysA, sqRelEstimates);
//}
return(ssqRelEstimates);
}
public EstimatorWrapper(string name, Dictionary<string, string> parameters)
{
this._judged = new Dictionary<string, RelevanceEstimate>();
this._name = name;
this._estimator = null;
this._parameters = parameters;
switch (this._name) {
case "uniform":
if (parameters.Count != 0) {
throw new ParseException("Estimator 'uniform' does not have parameters.");
}
break;
case "mout":
if (parameters.Count != 1 || !parameters.ContainsKey("meta")) {
throw new ParseException("Invalid parameters for estimator 'mout'.");
}
string metadataPath = parameters["meta"];
if (!File.Exists(metadataPath)) {
throw new ArgumentException("Metadata file '" + metadataPath + "' does not exist.");
}
break;
case "mjud":
if (!((parameters.Count == 1 && parameters.ContainsKey("meta")) ||
(parameters.Count == 2 && parameters.ContainsKey("meta") && parameters.ContainsKey("judged")))) {
throw new ParseException("Invalid parameters for estimator 'mjud'.");
}
metadataPath = parameters["meta"];
if (!File.Exists(metadataPath)) {
throw new ArgumentException("Metadata file '" + metadataPath + "' does not exist.");
}
if (parameters.Count == 2) {
string judgedPath = parameters["judged"];
if (!File.Exists(judgedPath)) {
throw new ArgumentException("Known judgments file '" + judgedPath + "' does not exist.");
}
}
break;
default:
throw new ParseException("'" + name + "' is not a valid estimator name.");
}
}
public RelativeEffectivenessEstimate Estimate(Run runA, Run runB, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
{
double e = 0, var = 0;
// Traverse docs retrieved by A
HashSet <string> inRunA = new HashSet <string>(); // retrieved by run A
foreach (string doc in runA.Documents)
{
RelevanceEstimate docEst = relEstimator.Estimate(runA.Query, doc);
e += docEst.Expectation;
var += docEst.Variance;
inRunA.Add(doc);
}
// Traverse docs retrieved by B
foreach (string doc in runB.Documents)
{
RelevanceEstimate docEst = relEstimator.Estimate(runB.Query, doc);
e -= docEst.Expectation;
if (inRunA.Contains(doc))
{
// If retrieved in both runs, does not contribute to variance
var -= docEst.Variance;
}
else
{
var += docEst.Variance;
}
}
// Compute average
e /= inRunA.Count;
var /= inRunA.Count * inRunA.Count;
// Normalize between 0 and 1
e /= this.MaxRelevance;
var /= this.MaxRelevance * this.MaxRelevance;
Estimate est = new Estimate(e, var);
return(new RelativeEffectivenessEstimate(runA.System, runB.System, runA.Query,
e, var,
confEstimator.EstimateInterval(est), confEstimator.EstimateRelativeConfidence(est)));
}
public AbsoluteEffectivenessEstimate Estimate(Run run, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
{
double e = 0, var = 0;
// Traverse docs retrieved
foreach (string doc in run.Documents)
{
RelevanceEstimate docEst = relEstimator.Estimate(run.Query, doc);
e += docEst.Expectation;
var += docEst.Variance;
}
// Compute average
e /= run.Documents.Count();
var /= run.Documents.Count() * run.Documents.Count();
// Normalize between 0 and 1
e /= this.MaxRelevance;
var /= this.MaxRelevance * this.MaxRelevance;
Estimate est = new Estimate(e, var);
return(new AbsoluteEffectivenessEstimate(run.System, run.Query,
e, var,
confEstimator.EstimateInterval(est), confEstimator.EstimateAbsoluteConfidence(est)));
}
public MoutRelevanceEstimator(IEnumerable <Run> runs, IEnumerable <Metadata> metadata)
{
// Instantiate model: fSYS, OV, fSYS:OV, fART, sGEN, fGEN, sGEN:fGEN
this._model = new OrdinalLogisticRegression(MoutRelevanceEstimator.LABELS, MoutRelevanceEstimator.ALPHAS, MoutRelevanceEstimator.BETAS);
this._defaultEstimator = new UniformRelevanceEstimator(100);
// Number of systems and metadata
int nSys = runs.Select(r => r.System).Distinct().Count();
Dictionary <string, string> artists = new Dictionary <string, string>(); // [doc, artist]
Dictionary <string, string> genres = new Dictionary <string, string>(); // [doc, genre]
foreach (var m in metadata)
{
artists[m.Document] = m.Artist;
genres[m.Document] = m.Genre;
}
// Auxiliary structure for easier computation of fGEN and fART
Dictionary <string, HashSet <string> > qDocs = new Dictionary <string, HashSet <string> >();
// fSYS and OV
this._fSYS = new Dictionary <string, double>();
foreach (var run in runs)
{
string query = run.Query;
HashSet <string> docs = null;
if (!qDocs.TryGetValue(query, out docs))
{
docs = new HashSet <string>();
qDocs.Add(query, docs);
}
foreach (string doc in run.Documents)
{
string id = RelevanceEstimate.GetId(query, doc);
// fSYS
double fSYS = 0;
this._fSYS.TryGetValue(id, out fSYS);
this._fSYS[id] = fSYS + 1.0 / nSys;
docs.Add(doc);
}
}
// OV
this._OV = ((double)this._fSYS.Count) / (nSys * qDocs.Count * runs.First().Documents.Count());
// sGEN, fGEN and fART, traverse qDocs
this._sGEN = new Dictionary <string, bool>();
this._fGEN = new Dictionary <string, double>();
this._fART = new Dictionary <string, double>();
foreach (var docs in qDocs)
{
string query = docs.Key;
foreach (string doc in docs.Value)
{
string id = RelevanceEstimate.GetId(query, doc);
// sGEN and fGEN
if (genres.ContainsKey(doc))
{
string docGEN = genres[doc];
// sGEN
if (genres.ContainsKey(query))
{
this._sGEN[id] = docGEN == genres[query];
}
// fGEN
double docfGEN = 0;
int docfGENnotnull = 0;
// traverse all documents individually
foreach (string doc2 in docs.Value)
{
if (genres.ContainsKey(doc2))
{
string doc2GEN = genres[doc2];
if (docGEN == doc2GEN)
{
docfGEN++;
}
docfGENnotnull++;
}
}
this._fGEN[id] = docfGEN / docfGENnotnull;
}
// fART
if (artists.ContainsKey(doc))
{
string docART = artists[doc];
double docfART = 0;
int docfARTnotnull = 0;
// traverse all documents individually
foreach (string doc2 in docs.Value)
{
if (artists.ContainsKey(doc2))
{
string doc2ART = artists[doc2];
if (docART == doc2ART)
{
docfART++;
}
docfARTnotnull++;
}
}
this._fART[id] = docfART / docfARTnotnull;
}
}
}
}
internal static Dictionary<string, Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>>> GetSystemSystemQueryRelatives(
Dictionary<string, Dictionary<string, Run>> sqRuns,
IMeasure measure, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
{
Dictionary<string, Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>>> ssqRelEstimates =
new Dictionary<string, Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>>>(); // [sysA [sysB [query rel]]]
string[] allSystems = sqRuns.Keys.ToArray();
Parallel.For(0, allSystems.Length - 1, i => {
string sysA = allSystems[i];
var runsA = sqRuns[sysA];
Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>> sqRelEstimates = new Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>>();
for (int j = i + 1; j < allSystems.Length; j++) {
Dictionary<string, RelativeEffectivenessEstimate> qRelEstimates = new Dictionary<string, RelativeEffectivenessEstimate>();
string sysB = allSystems[j];
var runsB = sqRuns[sysB];
foreach (var qRun in runsA) {
qRelEstimates.Add(qRun.Key, measure.Estimate(qRun.Value, runsB[qRun.Key], relEstimator, confEstimator));
}
sqRelEstimates.Add(sysB, qRelEstimates);
}
lock (ssqRelEstimates) {
ssqRelEstimates.Add(sysA, sqRelEstimates);
}
});
//for (int i = 0; i < allSystems.Length - 1; i++) {
// string sysA = allSystems[i];
// var runsA = sqRuns[sysA];
// Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>> sqRelEstimates = new Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>>();
// for (int j = i + 1; j < allSystems.Length; j++) {
// Dictionary<string, RelativeEffectivenessEstimate> qRelEstimates = new Dictionary<string, RelativeEffectivenessEstimate>();
// string sysB = allSystems[j];
// var runsB = sqRuns[sysB];
// foreach (var qRun in runsA) {
// qRelEstimates.Add(qRun.Key, measure.Estimate(qRun.Value, runsB[qRun.Key], relEstimator, confEstimator));
// }
// sqRelEstimates.Add(sysB, qRelEstimates);
// }
// ssqRelEstimates.Add(sysA, sqRelEstimates);
//}
return ssqRelEstimates;
}
internal static Dictionary<string, Dictionary<string, AbsoluteEffectivenessEstimate>> GetSystemQueryAbsolutes(
Dictionary<string, Dictionary<string, Run>> sqRuns,
IMeasure measure, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
{
Dictionary<string, Dictionary<string, AbsoluteEffectivenessEstimate>> sqAbss = new Dictionary<string, Dictionary<string, AbsoluteEffectivenessEstimate>>();
foreach (var sqRun in sqRuns) {
Dictionary<string, AbsoluteEffectivenessEstimate> qAbs = new Dictionary<string, AbsoluteEffectivenessEstimate>();
foreach (var qRun in sqRun.Value) {
qAbs.Add(qRun.Key, measure.Estimate(qRun.Value, relEstimator, confEstimator));
}
sqAbss.Add(sqRun.Key, qAbs);
}
return sqAbss;
}
public void Initialize(IEnumerable<Run> runs, IEnumerable<RelevanceEstimate> judged)
{
// Re-structure known judgments
foreach (var j in judged) {
string id = RelevanceEstimate.GetId(j.Query, j.Document);
this._judged[id] = j;
}
// Instantiate estimator
switch (this._name) {
case "uniform":
// nothing to initialize
this._estimator = new UniformRelevanceEstimator(100);
break;
case "mout":
// read metadata
IEnumerable<Metadata> metadata = AbstractCommand.ReadMetadata(this._parameters["meta"]);
this._estimator = new MoutRelevanceEstimator(runs, metadata);
break;
case "mjud":
// read metadata
metadata = AbstractCommand.ReadMetadata(this._parameters["meta"]);
IEnumerable<RelevanceEstimate> judgedEst = this._parameters.ContainsKey("judged") ?
AbstractCommand.ReadKnownJudgments(this._parameters["judged"]) :
new RelevanceEstimate[] { };
this._estimator = new MjudRelevanceEstimator(runs, metadata, judgedEst);
break;
}
}
protected Dictionary<string, bool> _sGEN; // [querydoc, sGEN]
#endregion Fields
#region Constructors
public MoutRelevanceEstimator(IEnumerable<Run> runs, IEnumerable<Metadata> metadata)
{
// Instantiate model: fSYS, OV, fSYS:OV, fART, sGEN, fGEN, sGEN:fGEN
this._model = new OrdinalLogisticRegression(MoutRelevanceEstimator.LABELS, MoutRelevanceEstimator.ALPHAS, MoutRelevanceEstimator.BETAS);
this._defaultEstimator = new UniformRelevanceEstimator(100);
// Number of systems and metadata
int nSys = runs.Select(r => r.System).Distinct().Count();
Dictionary<string, string> artists = new Dictionary<string, string>();// [doc, artist]
Dictionary<string, string> genres = new Dictionary<string, string>();// [doc, genre]
foreach (var m in metadata) {
artists[m.Document] = m.Artist;
genres[m.Document] = m.Genre;
}
// Auxiliary structure for easier computation of fGEN and fART
Dictionary<string, HashSet<string>> qDocs = new Dictionary<string, HashSet<string>>();
// fSYS and OV
this._fSYS = new Dictionary<string, double>();
foreach (var run in runs) {
string query = run.Query;
HashSet<string> docs = null;
if (!qDocs.TryGetValue(query, out docs)) {
docs = new HashSet<string>();
qDocs.Add(query, docs);
}
foreach (string doc in run.Documents) {
string id = RelevanceEstimate.GetId(query, doc);
// fSYS
double fSYS = 0;
this._fSYS.TryGetValue(id, out fSYS);
this._fSYS[id] = fSYS + 1.0 / nSys;
docs.Add(doc);
}
}
// OV
this._OV = ((double)this._fSYS.Count) / (nSys * qDocs.Count * runs.First().Documents.Count());
// sGEN, fGEN and fART, traverse qDocs
this._sGEN = new Dictionary<string, bool>();
this._fGEN = new Dictionary<string, double>();
this._fART = new Dictionary<string, double>();
foreach (var docs in qDocs) {
string query = docs.Key;
foreach (string doc in docs.Value) {
string id = RelevanceEstimate.GetId(query, doc);
// sGEN and fGEN
if (genres.ContainsKey(doc)) {
string docGEN = genres[doc];
// sGEN
if (genres.ContainsKey(query)) {
this._sGEN[id] = docGEN == genres[query];
}
// fGEN
double docfGEN = 0;
int docfGENnotnull = 0;
// traverse all documents individually
foreach (string doc2 in docs.Value) {
if (genres.ContainsKey(doc2)) {
string doc2GEN = genres[doc2];
if (docGEN == doc2GEN) {
docfGEN++;
}
docfGENnotnull++;
}
}
this._fGEN[id] = docfGEN / docfGENnotnull;
}
// fART
if (artists.ContainsKey(doc)) {
string docART = artists[doc];
double docfART = 0;
int docfARTnotnull = 0;
// traverse all documents individually
foreach (string doc2 in docs.Value) {
if (artists.ContainsKey(doc2)) {
string doc2ART = artists[doc2];
if (docART == doc2ART) {
docfART++;
}
docfARTnotnull++;
}
}
this._fART[id] = docfART / docfARTnotnull;
}
}
}
}
public AbsoluteEffectivenessEstimate Estimate(Run run, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
{
double e = 0, var = 0;
// Traverse docs retrieved
foreach (string doc in run.Documents) {
RelevanceEstimate docEst = relEstimator.Estimate(run.Query, doc);
e += docEst.Expectation;
var += docEst.Variance;
}
// Compute average
e /= run.Documents.Count();
var /= run.Documents.Count() * run.Documents.Count();
// Normalize between 0 and 1
e /= this.MaxRelevance;
var /= this.MaxRelevance * this.MaxRelevance;
Estimate est = new Estimate(e, var);
return new AbsoluteEffectivenessEstimate(run.System, run.Query,
e, var,
confEstimator.EstimateInterval(est), confEstimator.EstimateAbsoluteConfidence(est));
}
public RelativeEffectivenessEstimate Estimate(Run runA, Run runB, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
{
double e = 0, var = 0;
// Traverse docs retrieved by A
HashSet<string> inRunA = new HashSet<string>(); // retrieved by run A
foreach (string doc in runA.Documents) {
RelevanceEstimate docEst = relEstimator.Estimate(runA.Query, doc);
e += docEst.Expectation;
var += docEst.Variance;
inRunA.Add(doc);
}
// Traverse docs retrieved by B
foreach (string doc in runB.Documents) {
RelevanceEstimate docEst = relEstimator.Estimate(runB.Query, doc);
e -= docEst.Expectation;
if (inRunA.Contains(doc)) {
// If retrieved in both runs, does not contribute to variance
var -= docEst.Variance;
} else {
var += docEst.Variance;
}
}
// Compute average
e /= inRunA.Count;
var /= inRunA.Count * inRunA.Count;
// Normalize between 0 and 1
e /= this.MaxRelevance;
var /= this.MaxRelevance * this.MaxRelevance;
Estimate est = new Estimate(e, var);
return new RelativeEffectivenessEstimate(runA.System, runB.System, runA.Query,
e, var,
confEstimator.EstimateInterval(est), confEstimator.EstimateRelativeConfidence(est));
}