/// <summary> /// Run the quality benchmark. /// </summary> /// <param name="judge"> /// The judge that can tell if a certain result doc is relevant for a certain quality query. /// If null, no judgements would be made. Usually null for a submission run. /// </param> /// <param name="submitRep">Submission report is created if non null.</param> /// <param name="qualityLog">If not null, quality run data would be printed for each query.</param> /// <returns><see cref="QualityStats"/> of each quality query that was executed.</returns> /// <exception cref="Exception">If quality benchmark failed to run.</exception> public virtual QualityStats[] Execute(IJudge judge, SubmissionReport submitRep, TextWriter qualityLog) { int nQueries = Math.Min(maxQueries, m_qualityQueries.Length); QualityStats[] stats = new QualityStats[nQueries]; for (int i = 0; i < nQueries; i++) { QualityQuery qq = m_qualityQueries[i]; // generate query Query q = m_qqParser.Parse(qq); // search with this query long t1 = J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results TopDocs td = m_searcher.Search(q, null, maxResults); long searchTime = (J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) - t1; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results //most likely we either submit or judge, but check both if (judge != null) { stats[i] = AnalyzeQueryResults(qq, q, td, judge, qualityLog, searchTime); } if (submitRep != null) { submitRep.Report(qq, td, m_docNameField, m_searcher); } } if (submitRep != null) { submitRep.Flush(); } return(stats); }
public void TestTrecTopicsReader() { // prepare topics Stream topicsFile = GetType().getResourceAsStream("trecTopics.txt"); TrecTopicsReader qReader = new TrecTopicsReader(); QualityQuery[] qqs = qReader.ReadQueries( new StreamReader(topicsFile, Encoding.UTF8)); assertEquals(20, qqs.Length); QualityQuery qq = qqs[0]; assertEquals("statement months total 1987", qq.GetValue("title")); assertEquals("Topic 0 Description Line 1 Topic 0 Description Line 2", qq.GetValue("description")); assertEquals("Topic 0 Narrative Line 1 Topic 0 Narrative Line 2", qq.GetValue("narrative")); qq = qqs[1]; assertEquals("agreed 15 against five", qq.GetValue("title")); assertEquals("Topic 1 Description Line 1 Topic 1 Description Line 2", qq.GetValue("description")); assertEquals("Topic 1 Narrative Line 1 Topic 1 Narrative Line 2", qq.GetValue("narrative")); qq = qqs[19]; assertEquals("20 while common week", qq.GetValue("title")); assertEquals("Topic 19 Description Line 1 Topic 19 Description Line 2", qq.GetValue("description")); assertEquals("Topic 19 Narrative Line 1 Topic 19 Narrative Line 2", qq.GetValue("narrative")); }
/// <summary>Analyze/judge results for a single quality query; optionally log them.</summary> private QualityStats AnalyzeQueryResults(QualityQuery qq, Query q, TopDocs td, IJudge judge, TextWriter logger, long searchTime) { QualityStats stts = new QualityStats(judge.MaxRecall(qq), searchTime); ScoreDoc[] sd = td.ScoreDocs; long t1 = Support.Time.CurrentTimeMilliseconds(); // extraction of first doc name we measure also construction of doc name extractor, just in case. DocNameExtractor xt = new DocNameExtractor(m_docNameField); for (int i = 0; i < sd.Length; i++) { string docName = xt.DocName(m_searcher, sd[i].Doc); long docNameExtractTime = Support.Time.CurrentTimeMilliseconds() - t1; t1 = Support.Time.CurrentTimeMilliseconds(); bool isRelevant = judge.IsRelevant(docName, qq); stts.AddResult(i + 1, isRelevant, docNameExtractTime); } if (logger != null) { logger.WriteLine(qq.QueryID + " - " + q); stts.Log(qq.QueryID + " Stats:", 1, logger, " "); } return(stts); }