public override void Run()
{
// Read files
IEnumerable <Run> runs = AbstractCommand.ReadInputFile(this._inputPath);
IEnumerable <RelevanceEstimate> judged = new RelevanceEstimate[] { };
if (this._judgedPath != null)
{
judged = AbstractCommand.ReadKnownJudgments(this._judgedPath);
}
// Initialize wrapped estimator
this._estimator.Initialize(runs, judged);
// Compile list of all query-doc pairs
Dictionary <string, HashSet <string> > querydocs = AbstractCommand.ToQueryDocuments(runs);
// Estimate relevance of all query-doc pairs
List <RelevanceEstimate> estimates = new List <RelevanceEstimate>();
foreach (var qd in querydocs)
{
foreach (var doc in qd.Value)
{
estimates.Add(this._estimator.Estimate(qd.Key, doc));
}
}
// Output estimates
TabSeparated io = new TabSeparated(this._decimalDigits);
((IWriter <RelevanceEstimate>)io).Write(Console.Out, estimates);
}
internal static IEnumerable <Metadata> ReadMetadata(string file)
{
IEnumerable <Metadata> metadata;
try {
IReader <Metadata> reader = new TabSeparated();
using (StreamReader sr = new StreamReader(File.OpenRead(file))) {
metadata = reader.Read(sr);
}
} catch (Exception ex) {
throw new FormatException("Error reading metadata file: " + ex.Message, ex);
}
return(metadata);
}
internal static IEnumerable <RelevanceEstimate> ReadEstimatedJudgments(string file)
{
IEnumerable <RelevanceEstimate> estimates = null;
try {
IReader <RelevanceEstimate> runReader = new TabSeparated();
using (StreamReader sr = new StreamReader(File.OpenRead(file))) {
estimates = runReader.Read(sr);
}
} catch (Exception ex) {
throw new FormatException("Error reading estimated judgments file: " + ex.Message, ex);
}
return(estimates);
}
// Read files
internal static IEnumerable <Run> ReadInputFile(string file)
{
IEnumerable <Run> runs = null;
try {
IReader <Run> runReader = new TabSeparated();
using (StreamReader sr = new StreamReader(File.OpenRead(file))) {
runs = runReader.Read(sr);
}
} catch (Exception ex) {
throw new FormatException("Error reading input file: " + ex.Message, ex);
}
return(runs);
}
public override void Run()
{
// Read files
IEnumerable <Run> runs = AbstractCommand.ReadInputFile(this._inputPath);
IEnumerable <RelevanceEstimate> judged = new RelevanceEstimate[] { };
if (this._judgedPath != null)
{
judged = AbstractCommand.ReadKnownJudgments(this._judgedPath);
}
IEnumerable <RelevanceEstimate> estimates = AbstractCommand.ReadEstimatedJudgments(this._estimatedPath);
// Instantiate estimate store and measure
RelevanceEstimateStore store = new RelevanceEstimateStore(estimates);
store.Update(judged);
IMeasure measure = new CG(100); //TODO: max relevance
// Re-structure runs for efficient access
Dictionary <string, Dictionary <string, Run> > sqRuns = AbstractCommand.ToSystemQueryRuns(runs);
// Estimate per-query absolute effectiveness
Dictionary <string, Dictionary <string, AbsoluteEffectivenessEstimate> > sqAbss =
EvaluateCommand.GetSystemQueryAbsolutes(sqRuns, measure, store, this._confEstimator);
// Average and sort
List <AbsoluteEffectivenessEstimate> absSorted = EvaluateCommand.GetSortedMeanAbsolutes(sqAbss, this._confEstimator);
// Estimate per-query relative effectiveness
Dictionary <string, Dictionary <string, Dictionary <string, RelativeEffectivenessEstimate> > > ssqRels =
EvaluateCommand.GetSystemSystemQueryRelatives(sqRuns, measure, store, this._confEstimator);
// Average (already sorted)
List <RelativeEffectivenessEstimate> relSorted = EvaluateCommand.GetSortedMeanRelatives(ssqRels, this._confEstimator);
// Output estimates
TabSeparated io = new TabSeparated(this._decimalDigits);
((IWriter <AbsoluteEffectivenessEstimate>)io).Write(Console.Out, absSorted);
((IWriter <RelativeEffectivenessEstimate>)io).Write(Console.Out, relSorted);
}
public override void Run()
{
// Read files
IEnumerable<Run> runs = AbstractCommand.ReadInputFile(this._inputPath);
IEnumerable<RelevanceEstimate> judged = new RelevanceEstimate[] { };
if (this._judgedPath != null) {
judged = AbstractCommand.ReadKnownJudgments(this._judgedPath);
}
// Initialize wrapped estimator
this._estimator.Initialize(runs, judged);
// Compile list of all query-doc pairs
Dictionary<string, HashSet<string>> querydocs = AbstractCommand.ToQueryDocuments(runs);
// Estimate relevance of all query-doc pairs
List<RelevanceEstimate> estimates = new List<RelevanceEstimate>();
foreach (var qd in querydocs) {
foreach (var doc in qd.Value) {
estimates.Add(this._estimator.Estimate(qd.Key, doc));
}
}
// Output estimates
TabSeparated io = new TabSeparated(this._decimalDigits);
((IWriter<RelevanceEstimate>)io).Write(Console.Out, estimates);
}
public override void Run()
{
// Read files
IEnumerable<Run> runs = AbstractCommand.ReadInputFile(this._inputPath);
IEnumerable<RelevanceEstimate> judged = new RelevanceEstimate[] { };
if (this._judgedPath != null) {
judged = AbstractCommand.ReadKnownJudgments(this._judgedPath);
}
IEnumerable<RelevanceEstimate> estimates = AbstractCommand.ReadEstimatedJudgments(this._estimatedPath);
// Instantiate estimate store and measure
RelevanceEstimateStore store = new RelevanceEstimateStore(estimates);
store.Update(judged);
IMeasure measure = new CG(100); //TODO: max relevance
// Re-structure runs for efficient access
Dictionary<string, Dictionary<string, Run>> sqRuns = AbstractCommand.ToSystemQueryRuns(runs);
// Estimate per-query absolute effectiveness
Dictionary<string, Dictionary<string, AbsoluteEffectivenessEstimate>> sqAbss =
EvaluateCommand.GetSystemQueryAbsolutes(sqRuns, measure, store, this._confEstimator);
// Average and sort
List<AbsoluteEffectivenessEstimate> absSorted = EvaluateCommand.GetSortedMeanAbsolutes(sqAbss, this._confEstimator);
// Estimate per-query relative effectiveness
Dictionary<string, Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>>> ssqRels =
EvaluateCommand.GetSystemSystemQueryRelatives(sqRuns, measure, store, this._confEstimator);
// Average (already sorted)
List<RelativeEffectivenessEstimate> relSorted = EvaluateCommand.GetSortedMeanRelatives(ssqRels, this._confEstimator);
// Output estimates
TabSeparated io = new TabSeparated(this._decimalDigits);
((IWriter<AbsoluteEffectivenessEstimate>)io).Write(Console.Out, absSorted);
((IWriter<RelativeEffectivenessEstimate>)io).Write(Console.Out, relSorted);
}
internal static IEnumerable<Metadata> ReadMetadata(string file)
{
IEnumerable<Metadata> metadata;
try {
IReader<Metadata> reader = new TabSeparated();
using (StreamReader sr = new StreamReader(File.OpenRead(file))) {
metadata = reader.Read(sr);
}
} catch (Exception ex) {
throw new FormatException("Error reading metadata file: " + ex.Message, ex);
}
return metadata;
}
internal static IEnumerable<RelevanceEstimate> ReadKnownJudgments(string file)
{
IEnumerable<RelevanceEstimate> judged = null;
try {
IReader<RelevanceEstimate> runReader = new TabSeparated();
using (StreamReader sr = new StreamReader(File.OpenRead(file))) {
judged = runReader.Read(sr);
}
} catch (Exception ex) {
throw new FormatException("Error reading known judgments file: " + ex.Message, ex);
}
return judged;
}
// Read files
internal static IEnumerable<Run> ReadInputFile(string file)
{
IEnumerable<Run> runs = null;
try {
IReader<Run> runReader = new TabSeparated();
using (StreamReader sr = new StreamReader(File.OpenRead(file))) {
runs = runReader.Read(sr);
}
} catch (Exception ex) {
throw new FormatException("Error reading input file: " + ex.Message, ex);
}
return runs;
}
public override void Run()
{
// Read files
IEnumerable <Run> runs = AbstractCommand.ReadInputFile(this._inputPath);
IEnumerable <RelevanceEstimate> judged = new RelevanceEstimate[] { };
if (this._judgedPath != null)
{
judged = AbstractCommand.ReadKnownJudgments(this._judgedPath);
}
IEnumerable <RelevanceEstimate> estimates = AbstractCommand.ReadEstimatedJudgments(this._estimatedPath);
// Instantiate estimate store and measure
RelevanceEstimateStore store = new RelevanceEstimateStore(estimates);
store.Update(judged);
IMeasure measure = new CG(100); //TODO: max relevance
// Compile list of all query-doc-sys-rank tuples
Dictionary <string, Dictionary <string, Dictionary <string, int> > > qdsRanks = AbstractCommand.ToQueryDocumentSystemRanks(runs);
// Re-structure estimates
Dictionary <string, Dictionary <string, RelevanceEstimate> > qdEstimates = new Dictionary <string, Dictionary <string, RelevanceEstimate> >();
foreach (var est in estimates)
{
Dictionary <string, RelevanceEstimate> dEstimates = null;
if (!qdEstimates.TryGetValue(est.Query, out dEstimates))
{
dEstimates = new Dictionary <string, RelevanceEstimate>();
qdEstimates.Add(est.Query, dEstimates);
}
dEstimates.Add(est.Document, est);
}
// Remove judged query-docs
foreach (var j in judged)
{
Dictionary <string, RelevanceEstimate> dEstimates = null;
if (qdEstimates.TryGetValue(j.Query, out dEstimates))
{
dEstimates.Remove(j.Document);
if (dEstimates.Count == 0)
{
qdEstimates.Remove(j.Query);
}
}
}
bool needsNext = false;
// Re-structure runs for efficient access
Dictionary <string, Dictionary <string, Run> > sqRuns = AbstractCommand.ToSystemQueryRuns(runs);
if (this._target == EvaluationTargets.Relative)
{
// Estimate per-query relative effectiveness
Dictionary <string, Dictionary <string, Dictionary <string, RelativeEffectivenessEstimate> > > ssqRels =
EvaluateCommand.GetSystemSystemQueryRelatives(sqRuns, measure, store, this._confEstimator);
// Average (already sorted)
List <RelativeEffectivenessEstimate> relSorted = EvaluateCommand.GetSortedMeanRelatives(ssqRels, this._confEstimator);
if (relSorted.Average(r => r.Confidence) < this._confidence)
{
needsNext = true;
measure.ComputeQueryDocumentWeights(qdEstimates, qdsRanks, ssqRels);
}
}
else
{
// Estimate per-query absolute effectiveness
Dictionary <string, Dictionary <string, AbsoluteEffectivenessEstimate> > sqAbss =
EvaluateCommand.GetSystemQueryAbsolutes(sqRuns, measure, store, this._confEstimator);
// Average and sort
List <AbsoluteEffectivenessEstimate> absSorted = EvaluateCommand.GetSortedMeanAbsolutes(sqAbss, this._confEstimator);
if (absSorted.Average(a => a.Confidence) < this._confidence)
{
needsNext = true;
measure.ComputeQueryDocumentWeights(qdEstimates, qdsRanks, sqAbss);
}
}
if (needsNext)
{
var batches = NextCommand.GetBatches(qdEstimates, this._batchNum, this._batchSize);
TabSeparated io = new TabSeparated(this._decimalDigits);
((IWriter <List <RelevanceEstimate> >)io).Write(Console.Out, batches.Take(this._batchNum));
}
}
public override void Run()
{
// Read files
IEnumerable<Run> runs = AbstractCommand.ReadInputFile(this._inputPath);
IEnumerable<RelevanceEstimate> judged = new RelevanceEstimate[] { };
if (this._judgedPath != null) {
judged = AbstractCommand.ReadKnownJudgments(this._judgedPath);
}
IEnumerable<RelevanceEstimate> estimates = AbstractCommand.ReadEstimatedJudgments(this._estimatedPath);
// Instantiate estimate store and measure
RelevanceEstimateStore store = new RelevanceEstimateStore(estimates);
store.Update(judged);
IMeasure measure = new CG(100); //TODO: max relevance
// Compile list of all query-doc-sys-rank tuples
Dictionary<string, Dictionary<string, Dictionary<string, int>>> qdsRanks = AbstractCommand.ToQueryDocumentSystemRanks(runs);
// Re-structure estimates
Dictionary<string, Dictionary<string, RelevanceEstimate>> qdEstimates = new Dictionary<string, Dictionary<string, RelevanceEstimate>>();
foreach (var est in estimates) {
Dictionary<string, RelevanceEstimate> dEstimates = null;
if (!qdEstimates.TryGetValue(est.Query, out dEstimates)) {
dEstimates = new Dictionary<string, RelevanceEstimate>();
qdEstimates.Add(est.Query, dEstimates);
}
dEstimates.Add(est.Document, est);
}
// Remove judged query-docs
foreach (var j in judged) {
Dictionary<string, RelevanceEstimate> dEstimates = null;
if (qdEstimates.TryGetValue(j.Query, out dEstimates)) {
dEstimates.Remove(j.Document);
if (dEstimates.Count == 0) {
qdEstimates.Remove(j.Query);
}
}
}
bool needsNext = false;
// Re-structure runs for efficient access
Dictionary<string, Dictionary<string, Run>> sqRuns = AbstractCommand.ToSystemQueryRuns(runs);
if (this._target == EvaluationTargets.Relative) {
// Estimate per-query relative effectiveness
Dictionary<string, Dictionary<string, Dictionary<string, RelativeEffectivenessEstimate>>> ssqRels =
EvaluateCommand.GetSystemSystemQueryRelatives(sqRuns, measure, store, this._confEstimator);
// Average (already sorted)
List<RelativeEffectivenessEstimate> relSorted = EvaluateCommand.GetSortedMeanRelatives(ssqRels, this._confEstimator);
if (relSorted.Average(r => r.Confidence) < this._confidence) {
needsNext = true;
measure.ComputeQueryDocumentWeights(qdEstimates, qdsRanks, ssqRels);
}
} else {
// Estimate per-query absolute effectiveness
Dictionary<string, Dictionary<string, AbsoluteEffectivenessEstimate>> sqAbss =
EvaluateCommand.GetSystemQueryAbsolutes(sqRuns, measure, store, this._confEstimator);
// Average and sort
List<AbsoluteEffectivenessEstimate> absSorted = EvaluateCommand.GetSortedMeanAbsolutes(sqAbss, this._confEstimator);
if (absSorted.Average(a => a.Confidence) < this._confidence) {
needsNext = true;
measure.ComputeQueryDocumentWeights(qdEstimates, qdsRanks, sqAbss);
}
}
if (needsNext) {
var batches = NextCommand.GetBatches(qdEstimates, this._batchNum, this._batchSize);
TabSeparated io = new TabSeparated(this._decimalDigits);
((IWriter<List<RelevanceEstimate>>)io).Write(Console.Out, batches.Take(this._batchNum));
}
}