Exemple #1
0
        public void Initialize(IEnumerable <Run> runs, IEnumerable <RelevanceEstimate> judged)
        {
            // Re-structure known judgments
            foreach (var j in judged)
            {
                string id = RelevanceEstimate.GetId(j.Query, j.Document);
                this._judged[id] = j;
            }
            // Instantiate estimator
            switch (this._name)
            {
            case "uniform":
                // nothing to initialize
                this._estimator = new UniformRelevanceEstimator(100);
                break;

            case "mout":
                // read metadata
                IEnumerable <Metadata> metadata = AbstractCommand.ReadMetadata(this._parameters["meta"]);
                this._estimator = new MoutRelevanceEstimator(runs, metadata);
                break;

            case "mjud":
                // read metadata
                metadata = AbstractCommand.ReadMetadata(this._parameters["meta"]);
                IEnumerable <RelevanceEstimate> judgedEst = this._parameters.ContainsKey("judged") ?
                                                            AbstractCommand.ReadKnownJudgments(this._parameters["judged"]) :
                                                            new RelevanceEstimate[] { };
                this._estimator = new MjudRelevanceEstimator(runs, metadata, judgedEst);
                break;
            }
        }
Exemple #2
0
        public MjudRelevanceEstimator(IEnumerable <Run> runs, IEnumerable <Metadata> metadata, IEnumerable <RelevanceEstimate> judged)
        {
            // Instantiate model: fSYS, aSYS, aART
            this._model            = new OrdinalLogisticRegression(MjudRelevanceEstimator.LABELS, MjudRelevanceEstimator.ALPHAS, MjudRelevanceEstimator.BETAS);
            this._defaultEstimator = new MoutRelevanceEstimator(runs, metadata);

            // Number of systems and metadata
            int nSys = runs.Select(r => r.System).Distinct().Count();

            this._dArtists = new Dictionary <string, string>();
            foreach (var m in metadata)
            {
                this._dArtists[m.Document] = m.Artist;
            }
            // fSYS
            this._fSYS   = new Dictionary <string, double>();
            this._aSYS   = new Dictionary <string, double>();
            this._aART   = new Dictionary <string, double>();
            this._sRels  = new Dictionary <string, List <double> >();
            this._qaRels = new Dictionary <string, List <double> >();
            foreach (var run in runs)
            {
                string query = run.Query;
                foreach (string doc in run.Documents)
                {
                    string id = RelevanceEstimate.GetId(query, doc);
                    // fSYS
                    double fSYS = 0;
                    this._fSYS.TryGetValue(id, out fSYS);
                    this._fSYS[id] = fSYS + 1.0 / nSys;

                    this._aSYS[id] = 0;
                    this._aART[id] = 0;
                    // sRels
                    if (!this._sRels.ContainsKey(run.System))
                    {
                        this._sRels[run.System] = new List <double>();
                    }
                    // qaRels
                    string artist = null;
                    if (this._dArtists.TryGetValue(doc, out artist) && !this._qaRels.ContainsKey(query + "\t" + artist))
                    {
                        this._qaRels[query + "\t" + artist] = new List <double>();
                    }
                }
            }
            // OV
            this._OV = ((double)this._fSYS.Count) / (nSys * (runs.Count() / nSys) * runs.First().Documents.Count());

            this._qdsRanks = jurbano.Allcea.Cli.AbstractCommand.ToQueryDocumentSystemRanks(runs);

            // Incorporate known judgments
            foreach (var est in judged)
            {
                this.Update(est);
            }
            this._needsUpdate = true;
        }
Exemple #3
0
        public EstimatorWrapper(string name, Dictionary <string, string> parameters)
        {
            this._judged = new Dictionary <string, RelevanceEstimate>();

            this._name       = name;
            this._estimator  = null;
            this._parameters = parameters;
            switch (this._name)
            {
            case "uniform":
                if (parameters.Count != 0)
                {
                    throw new ParseException("Estimator 'uniform' does not have parameters.");
                }
                break;

            case "mout":
                if (parameters.Count != 1 || !parameters.ContainsKey("meta"))
                {
                    throw new ParseException("Invalid parameters for estimator 'mout'.");
                }
                string metadataPath = parameters["meta"];
                if (!File.Exists(metadataPath))
                {
                    throw new ArgumentException("Metadata file '" + metadataPath + "' does not exist.");
                }
                break;

            case "mjud":
                if (!((parameters.Count == 1 && parameters.ContainsKey("meta")) ||
                      (parameters.Count == 2 && parameters.ContainsKey("meta") && parameters.ContainsKey("judged"))))
                {
                    throw new ParseException("Invalid parameters for estimator 'mjud'.");
                }
                metadataPath = parameters["meta"];
                if (!File.Exists(metadataPath))
                {
                    throw new ArgumentException("Metadata file '" + metadataPath + "' does not exist.");
                }
                if (parameters.Count == 2)
                {
                    string judgedPath = parameters["judged"];
                    if (!File.Exists(judgedPath))
                    {
                        throw new ArgumentException("Known judgments file '" + judgedPath + "' does not exist.");
                    }
                }
                break;

            default:
                throw new ParseException("'" + name + "' is not a valid estimator name.");
            }
        }
        protected Dictionary<string, List<double>> _sRels; // [sys, [rel]]

        #endregion Fields

        #region Constructors

        public MjudRelevanceEstimator(IEnumerable<Run> runs, IEnumerable<Metadata> metadata, IEnumerable<RelevanceEstimate> judged)
        {
            // Instantiate model: fSYS, aSYS, aART
            this._model = new OrdinalLogisticRegression(MjudRelevanceEstimator.LABELS, MjudRelevanceEstimator.ALPHAS, MjudRelevanceEstimator.BETAS);
            this._defaultEstimator = new MoutRelevanceEstimator(runs, metadata);

            // Number of systems and metadata
            int nSys = runs.Select(r => r.System).Distinct().Count();
            this._dArtists = new Dictionary<string, string>();
            foreach (var m in metadata) {
                this._dArtists[m.Document] = m.Artist;
            }
            // fSYS
            this._fSYS = new Dictionary<string, double>();
            this._aSYS = new Dictionary<string, double>();
            this._aART = new Dictionary<string, double>();
            this._sRels = new Dictionary<string, List<double>>();
            this._qaRels = new Dictionary<string, List<double>>();
            foreach (var run in runs) {
                string query = run.Query;
                foreach (string doc in run.Documents) {
                    string id = RelevanceEstimate.GetId(query, doc);
                    // fSYS
                    double fSYS = 0;
                    this._fSYS.TryGetValue(id, out fSYS);
                    this._fSYS[id] = fSYS + 1.0 / nSys;

                    this._aSYS[id] = 0;
                    this._aART[id] = 0;
                    // sRels
                    if (!this._sRels.ContainsKey(run.System)) {
                        this._sRels[run.System] = new List<double>();
                    }
                    // qaRels
                    string artist = null;
                    if (this._dArtists.TryGetValue(doc, out artist) && !this._qaRels.ContainsKey(query + "\t" + artist)) {
                        this._qaRels[query + "\t" + artist] = new List<double>();
                    }
                }
            }
            // OV
            this._OV = ((double)this._fSYS.Count) / (nSys * (runs.Count()/nSys) * runs.First().Documents.Count());

            this._qdsRanks = jurbano.Allcea.Cli.AbstractCommand.ToQueryDocumentSystemRanks(runs);

            // Incorporate known judgments
            foreach (var est in judged) {
                this.Update(est);
            }
            this._needsUpdate = true;
        }
Exemple #5
0
        internal static Dictionary <string, Dictionary <string, AbsoluteEffectivenessEstimate> > GetSystemQueryAbsolutes(
            Dictionary <string, Dictionary <string, Run> > sqRuns,
            IMeasure measure, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
        {
            Dictionary <string, Dictionary <string, AbsoluteEffectivenessEstimate> > sqAbss = new Dictionary <string, Dictionary <string, AbsoluteEffectivenessEstimate> >();

            foreach (var sqRun in sqRuns)
            {
                Dictionary <string, AbsoluteEffectivenessEstimate> qAbs = new Dictionary <string, AbsoluteEffectivenessEstimate>();
                foreach (var qRun in sqRun.Value)
                {
                    qAbs.Add(qRun.Key, measure.Estimate(qRun.Value, relEstimator, confEstimator));
                }
                sqAbss.Add(sqRun.Key, qAbs);
            }
            return(sqAbss);
        }
Exemple #6
0
        internal static Dictionary <string, Dictionary <string, Dictionary <string, RelativeEffectivenessEstimate> > > GetSystemSystemQueryRelatives(
            Dictionary <string, Dictionary <string, Run> > sqRuns,
            IMeasure measure, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
        {
            Dictionary <string, Dictionary <string, Dictionary <string, RelativeEffectivenessEstimate> > > ssqRelEstimates =
                new Dictionary <string, Dictionary <string, Dictionary <string, RelativeEffectivenessEstimate> > >(); // [sysA [sysB [query rel]]]

            string[] allSystems = sqRuns.Keys.ToArray();

            Parallel.For(0, allSystems.Length - 1, i => {
                string sysA = allSystems[i];
                var runsA   = sqRuns[sysA];
                Dictionary <string, Dictionary <string, RelativeEffectivenessEstimate> > sqRelEstimates = new Dictionary <string, Dictionary <string, RelativeEffectivenessEstimate> >();
                for (int j = i + 1; j < allSystems.Length; j++)
                {
                    Dictionary <string, RelativeEffectivenessEstimate> qRelEstimates = new Dictionary <string, RelativeEffectivenessEstimate>();
                    string sysB = allSystems[j];
                    var runsB   = sqRuns[sysB];
                    foreach (var qRun in runsA)
                    {
                        qRelEstimates.Add(qRun.Key, measure.Estimate(qRun.Value, runsB[qRun.Key], relEstimator, confEstimator));
                    }
                    sqRelEstimates.Add(sysB, qRelEstimates);
                }
                lock (ssqRelEstimates) {
                    ssqRelEstimates.Add(sysA, sqRelEstimates);
                }
            });

            //for (int i = 0; i < allSystems.Length - 1; i++) {
            //    string sysA = allSystems[i];
            //    var runsA = sqRuns[sysA];
            //    Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>> sqRelEstimates = new Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>>();
            //    for (int j = i + 1; j < allSystems.Length; j++) {
            //        Dictionary<string, RelativeEffectivenessEstimate> qRelEstimates = new Dictionary<string, RelativeEffectivenessEstimate>();
            //        string sysB = allSystems[j];
            //        var runsB = sqRuns[sysB];
            //        foreach (var qRun in runsA) {
            //            qRelEstimates.Add(qRun.Key, measure.Estimate(qRun.Value, runsB[qRun.Key], relEstimator, confEstimator));
            //        }
            //        sqRelEstimates.Add(sysB, qRelEstimates);
            //    }
            //    ssqRelEstimates.Add(sysA, sqRelEstimates);
            //}
            return(ssqRelEstimates);
        }
Exemple #7
0
        public EstimatorWrapper(string name, Dictionary<string, string> parameters)
        {
            this._judged = new Dictionary<string, RelevanceEstimate>();

            this._name = name;
            this._estimator = null;
            this._parameters = parameters;
            switch (this._name) {
                case "uniform":
                    if (parameters.Count != 0) {
                        throw new ParseException("Estimator 'uniform' does not have parameters.");
                    }
                    break;
                case "mout":
                    if (parameters.Count != 1 || !parameters.ContainsKey("meta")) {
                        throw new ParseException("Invalid parameters for estimator 'mout'.");
                    }
                    string metadataPath = parameters["meta"];
                    if (!File.Exists(metadataPath)) {
                        throw new ArgumentException("Metadata file '" + metadataPath + "' does not exist.");
                    }
                    break;
                case "mjud":
                    if (!((parameters.Count == 1 && parameters.ContainsKey("meta")) ||
                          (parameters.Count == 2 && parameters.ContainsKey("meta") && parameters.ContainsKey("judged")))) {
                        throw new ParseException("Invalid parameters for estimator 'mjud'.");
                    }
                    metadataPath = parameters["meta"];
                    if (!File.Exists(metadataPath)) {
                        throw new ArgumentException("Metadata file '" + metadataPath + "' does not exist.");
                    }
                    if (parameters.Count == 2) {
                        string judgedPath = parameters["judged"];
                        if (!File.Exists(judgedPath)) {
                            throw new ArgumentException("Known judgments file '" + judgedPath + "' does not exist.");
                        }
                    }
                    break;
                default:
                    throw new ParseException("'" + name + "' is not a valid estimator name.");
            }
        }
Exemple #8
0
        public RelativeEffectivenessEstimate Estimate(Run runA, Run runB, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
        {
            double e = 0, var = 0;

            // Traverse docs retrieved by A
            HashSet <string> inRunA = new HashSet <string>(); // retrieved by run A

            foreach (string doc in runA.Documents)
            {
                RelevanceEstimate docEst = relEstimator.Estimate(runA.Query, doc);
                e   += docEst.Expectation;
                var += docEst.Variance;
                inRunA.Add(doc);
            }
            // Traverse docs retrieved by B
            foreach (string doc in runB.Documents)
            {
                RelevanceEstimate docEst = relEstimator.Estimate(runB.Query, doc);
                e -= docEst.Expectation;
                if (inRunA.Contains(doc))
                {
                    // If retrieved in both runs, does not contribute to variance
                    var -= docEst.Variance;
                }
                else
                {
                    var += docEst.Variance;
                }
            }
            // Compute average
            e   /= inRunA.Count;
            var /= inRunA.Count * inRunA.Count;
            // Normalize between 0 and 1
            e   /= this.MaxRelevance;
            var /= this.MaxRelevance * this.MaxRelevance;

            Estimate est            = new Estimate(e, var);

            return(new RelativeEffectivenessEstimate(runA.System, runB.System, runA.Query,
                                                     e, var,
                                                     confEstimator.EstimateInterval(est), confEstimator.EstimateRelativeConfidence(est)));
        }
Exemple #9
0
        public AbsoluteEffectivenessEstimate Estimate(Run run, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
        {
            double e = 0, var = 0;

            // Traverse docs retrieved
            foreach (string doc in run.Documents)
            {
                RelevanceEstimate docEst = relEstimator.Estimate(run.Query, doc);
                e   += docEst.Expectation;
                var += docEst.Variance;
            }
            // Compute average
            e   /= run.Documents.Count();
            var /= run.Documents.Count() * run.Documents.Count();
            // Normalize between 0 and 1
            e   /= this.MaxRelevance;
            var /= this.MaxRelevance * this.MaxRelevance;

            Estimate est = new Estimate(e, var);

            return(new AbsoluteEffectivenessEstimate(run.System, run.Query,
                                                     e, var,
                                                     confEstimator.EstimateInterval(est), confEstimator.EstimateAbsoluteConfidence(est)));
        }
Exemple #10
0
        public MoutRelevanceEstimator(IEnumerable <Run> runs, IEnumerable <Metadata> metadata)
        {
            // Instantiate model: fSYS, OV, fSYS:OV, fART, sGEN, fGEN, sGEN:fGEN
            this._model            = new OrdinalLogisticRegression(MoutRelevanceEstimator.LABELS, MoutRelevanceEstimator.ALPHAS, MoutRelevanceEstimator.BETAS);
            this._defaultEstimator = new UniformRelevanceEstimator(100);
            // Number of systems and metadata
            int nSys = runs.Select(r => r.System).Distinct().Count();
            Dictionary <string, string> artists = new Dictionary <string, string>(); // [doc, artist]
            Dictionary <string, string> genres  = new Dictionary <string, string>(); // [doc, genre]

            foreach (var m in metadata)
            {
                artists[m.Document] = m.Artist;
                genres[m.Document]  = m.Genre;
            }
            // Auxiliary structure for easier computation of fGEN and fART
            Dictionary <string, HashSet <string> > qDocs = new Dictionary <string, HashSet <string> >();

            // fSYS and OV
            this._fSYS = new Dictionary <string, double>();
            foreach (var run in runs)
            {
                string           query = run.Query;
                HashSet <string> docs  = null;
                if (!qDocs.TryGetValue(query, out docs))
                {
                    docs = new HashSet <string>();
                    qDocs.Add(query, docs);
                }
                foreach (string doc in run.Documents)
                {
                    string id = RelevanceEstimate.GetId(query, doc);
                    // fSYS
                    double fSYS = 0;
                    this._fSYS.TryGetValue(id, out fSYS);
                    this._fSYS[id] = fSYS + 1.0 / nSys;
                    docs.Add(doc);
                }
            }
            // OV
            this._OV = ((double)this._fSYS.Count) / (nSys * qDocs.Count * runs.First().Documents.Count());
            // sGEN, fGEN and fART, traverse qDocs
            this._sGEN = new Dictionary <string, bool>();
            this._fGEN = new Dictionary <string, double>();
            this._fART = new Dictionary <string, double>();
            foreach (var docs in qDocs)
            {
                string query = docs.Key;
                foreach (string doc in docs.Value)
                {
                    string id = RelevanceEstimate.GetId(query, doc);
                    // sGEN and fGEN
                    if (genres.ContainsKey(doc))
                    {
                        string docGEN = genres[doc];
                        // sGEN
                        if (genres.ContainsKey(query))
                        {
                            this._sGEN[id] = docGEN == genres[query];
                        }
                        // fGEN
                        double docfGEN        = 0;
                        int    docfGENnotnull = 0;
                        // traverse all documents individually
                        foreach (string doc2 in docs.Value)
                        {
                            if (genres.ContainsKey(doc2))
                            {
                                string doc2GEN = genres[doc2];
                                if (docGEN == doc2GEN)
                                {
                                    docfGEN++;
                                }
                                docfGENnotnull++;
                            }
                        }
                        this._fGEN[id] = docfGEN / docfGENnotnull;
                    }
                    // fART
                    if (artists.ContainsKey(doc))
                    {
                        string docART         = artists[doc];
                        double docfART        = 0;
                        int    docfARTnotnull = 0;
                        // traverse all documents individually
                        foreach (string doc2 in docs.Value)
                        {
                            if (artists.ContainsKey(doc2))
                            {
                                string doc2ART = artists[doc2];
                                if (docART == doc2ART)
                                {
                                    docfART++;
                                }
                                docfARTnotnull++;
                            }
                        }
                        this._fART[id] = docfART / docfARTnotnull;
                    }
                }
            }
        }
Exemple #11
0
        internal static Dictionary<string, Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>>> GetSystemSystemQueryRelatives(
            Dictionary<string, Dictionary<string, Run>> sqRuns,
            IMeasure measure, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
        {
            Dictionary<string, Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>>> ssqRelEstimates =
                new Dictionary<string, Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>>>(); // [sysA [sysB [query rel]]]
            string[] allSystems = sqRuns.Keys.ToArray();

            Parallel.For(0, allSystems.Length - 1, i => {
                string sysA = allSystems[i];
                var runsA = sqRuns[sysA];
                Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>> sqRelEstimates = new Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>>();
                for (int j = i + 1; j < allSystems.Length; j++) {
                    Dictionary<string, RelativeEffectivenessEstimate> qRelEstimates = new Dictionary<string, RelativeEffectivenessEstimate>();
                    string sysB = allSystems[j];
                    var runsB = sqRuns[sysB];
                    foreach (var qRun in runsA) {
                        qRelEstimates.Add(qRun.Key, measure.Estimate(qRun.Value, runsB[qRun.Key], relEstimator, confEstimator));
                    }
                    sqRelEstimates.Add(sysB, qRelEstimates);
                }
                lock (ssqRelEstimates) {
                    ssqRelEstimates.Add(sysA, sqRelEstimates);
                }
            });

            //for (int i = 0; i < allSystems.Length - 1; i++) {
            //    string sysA = allSystems[i];
            //    var runsA = sqRuns[sysA];
            //    Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>> sqRelEstimates = new Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>>();
            //    for (int j = i + 1; j < allSystems.Length; j++) {
            //        Dictionary<string, RelativeEffectivenessEstimate> qRelEstimates = new Dictionary<string, RelativeEffectivenessEstimate>();
            //        string sysB = allSystems[j];
            //        var runsB = sqRuns[sysB];
            //        foreach (var qRun in runsA) {
            //            qRelEstimates.Add(qRun.Key, measure.Estimate(qRun.Value, runsB[qRun.Key], relEstimator, confEstimator));
            //        }
            //        sqRelEstimates.Add(sysB, qRelEstimates);
            //    }
            //    ssqRelEstimates.Add(sysA, sqRelEstimates);
            //}
            return ssqRelEstimates;
        }
Exemple #12
0
 internal static Dictionary<string, Dictionary<string, AbsoluteEffectivenessEstimate>> GetSystemQueryAbsolutes(
     Dictionary<string, Dictionary<string, Run>> sqRuns,
     IMeasure measure, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
 {
     Dictionary<string, Dictionary<string, AbsoluteEffectivenessEstimate>> sqAbss = new Dictionary<string, Dictionary<string, AbsoluteEffectivenessEstimate>>();
     foreach (var sqRun in sqRuns) {
         Dictionary<string, AbsoluteEffectivenessEstimate> qAbs = new Dictionary<string, AbsoluteEffectivenessEstimate>();
         foreach (var qRun in sqRun.Value) {
             qAbs.Add(qRun.Key, measure.Estimate(qRun.Value, relEstimator, confEstimator));
         }
         sqAbss.Add(sqRun.Key, qAbs);
     }
     return sqAbss;
 }
Exemple #13
0
 public void Initialize(IEnumerable<Run> runs, IEnumerable<RelevanceEstimate> judged)
 {
     // Re-structure known judgments
     foreach (var j in judged) {
         string id = RelevanceEstimate.GetId(j.Query, j.Document);
         this._judged[id] = j;
     }
     // Instantiate estimator
     switch (this._name) {
         case "uniform":
             // nothing to initialize
             this._estimator = new UniformRelevanceEstimator(100);
             break;
         case "mout":
             // read metadata
             IEnumerable<Metadata> metadata = AbstractCommand.ReadMetadata(this._parameters["meta"]);
             this._estimator = new MoutRelevanceEstimator(runs, metadata);
             break;
         case "mjud":
             // read metadata
             metadata = AbstractCommand.ReadMetadata(this._parameters["meta"]);
             IEnumerable<RelevanceEstimate> judgedEst = this._parameters.ContainsKey("judged") ?
                 AbstractCommand.ReadKnownJudgments(this._parameters["judged"]) :
                 new RelevanceEstimate[] { };
             this._estimator = new MjudRelevanceEstimator(runs, metadata, judgedEst);
             break;
     }
 }
        protected Dictionary<string, bool> _sGEN; // [querydoc, sGEN]

        #endregion Fields

        #region Constructors

        public MoutRelevanceEstimator(IEnumerable<Run> runs, IEnumerable<Metadata> metadata)
        {
            // Instantiate model: fSYS, OV, fSYS:OV, fART, sGEN, fGEN, sGEN:fGEN
            this._model = new OrdinalLogisticRegression(MoutRelevanceEstimator.LABELS, MoutRelevanceEstimator.ALPHAS, MoutRelevanceEstimator.BETAS);
            this._defaultEstimator = new UniformRelevanceEstimator(100);
            // Number of systems and metadata
            int nSys = runs.Select(r => r.System).Distinct().Count();
            Dictionary<string, string> artists = new Dictionary<string, string>();// [doc, artist]
            Dictionary<string, string> genres = new Dictionary<string, string>();// [doc, genre]
            foreach (var m in metadata) {
                artists[m.Document] = m.Artist;
                genres[m.Document] = m.Genre;
            }
            // Auxiliary structure for easier computation of fGEN and fART
            Dictionary<string, HashSet<string>> qDocs = new Dictionary<string, HashSet<string>>();
            // fSYS and OV
            this._fSYS = new Dictionary<string, double>();
            foreach (var run in runs) {
                string query = run.Query;
                HashSet<string> docs = null;
                if (!qDocs.TryGetValue(query, out docs)) {
                    docs = new HashSet<string>();
                    qDocs.Add(query, docs);
                }
                foreach (string doc in run.Documents) {
                    string id = RelevanceEstimate.GetId(query, doc);
                    // fSYS
                    double fSYS = 0;
                    this._fSYS.TryGetValue(id, out fSYS);
                    this._fSYS[id] = fSYS + 1.0 / nSys;
                    docs.Add(doc);
                }
            }
            // OV
            this._OV = ((double)this._fSYS.Count) / (nSys * qDocs.Count * runs.First().Documents.Count());
            // sGEN, fGEN and fART, traverse qDocs
            this._sGEN = new Dictionary<string, bool>();
            this._fGEN = new Dictionary<string, double>();
            this._fART = new Dictionary<string, double>();
            foreach (var docs in qDocs) {
                string query = docs.Key;
                foreach (string doc in docs.Value) {
                    string id = RelevanceEstimate.GetId(query, doc);
                    // sGEN and fGEN
                    if (genres.ContainsKey(doc)) {
                        string docGEN = genres[doc];
                        // sGEN
                        if (genres.ContainsKey(query)) {
                            this._sGEN[id] = docGEN == genres[query];
                        }
                        // fGEN
                        double docfGEN = 0;
                        int docfGENnotnull = 0;
                        // traverse all documents individually
                        foreach (string doc2 in docs.Value) {
                            if (genres.ContainsKey(doc2)) {
                                string doc2GEN = genres[doc2];
                                if (docGEN == doc2GEN) {
                                    docfGEN++;
                                }
                                docfGENnotnull++;
                            }
                        }
                        this._fGEN[id] = docfGEN / docfGENnotnull;
                    }
                    // fART
                    if (artists.ContainsKey(doc)) {
                        string docART = artists[doc];
                        double docfART = 0;
                        int docfARTnotnull = 0;
                        // traverse all documents individually
                        foreach (string doc2 in docs.Value) {
                            if (artists.ContainsKey(doc2)) {
                                string doc2ART = artists[doc2];
                                if (docART == doc2ART) {
                                    docfART++;
                                }
                                docfARTnotnull++;
                            }
                        }
                        this._fART[id] = docfART / docfARTnotnull;
                    }
                }
            }
        }
Exemple #15
0
        public AbsoluteEffectivenessEstimate Estimate(Run run, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
        {
            double e = 0, var = 0;

            // Traverse docs retrieved
            foreach (string doc in run.Documents) {
                RelevanceEstimate docEst = relEstimator.Estimate(run.Query, doc);
                e += docEst.Expectation;
                var += docEst.Variance;
            }
            // Compute average
            e /= run.Documents.Count();
            var /= run.Documents.Count() * run.Documents.Count();
            // Normalize between 0 and 1
            e /= this.MaxRelevance;
            var /= this.MaxRelevance * this.MaxRelevance;

            Estimate est = new Estimate(e, var);

            return new AbsoluteEffectivenessEstimate(run.System, run.Query,
                e, var,
                confEstimator.EstimateInterval(est), confEstimator.EstimateAbsoluteConfidence(est));
        }
Exemple #16
0
        public RelativeEffectivenessEstimate Estimate(Run runA, Run runB, IRelevanceEstimator relEstimator, IConfidenceEstimator confEstimator)
        {
            double e = 0, var = 0;

            // Traverse docs retrieved by A
            HashSet<string> inRunA = new HashSet<string>(); // retrieved by run A
            foreach (string doc in runA.Documents) {
                RelevanceEstimate docEst = relEstimator.Estimate(runA.Query, doc);
                e += docEst.Expectation;
                var += docEst.Variance;
                inRunA.Add(doc);
            }
            // Traverse docs retrieved by B
            foreach (string doc in runB.Documents) {
                RelevanceEstimate docEst = relEstimator.Estimate(runB.Query, doc);
                e -= docEst.Expectation;
                if (inRunA.Contains(doc)) {
                    // If retrieved in both runs, does not contribute to variance
                    var -= docEst.Variance;
                } else {
                    var += docEst.Variance;
                }
            }
            // Compute average
            e /= inRunA.Count;
            var /= inRunA.Count * inRunA.Count;
            // Normalize between 0 and 1
            e /= this.MaxRelevance;
            var /= this.MaxRelevance * this.MaxRelevance;

            Estimate est = new Estimate(e, var);

            return new RelativeEffectivenessEstimate(runA.System, runB.System, runA.Query,
                e, var,
                confEstimator.EstimateInterval(est), confEstimator.EstimateRelativeConfidence(est));
        }