/// <summary>
/// Search shell (not interactive at this level)
/// </summary>
public static void Search(Index index, IEnumerable<CommandQuery> qReader, ShellSearchOptions searchOps)
{
BinaryWriter ResultOutput = null;
if (searchOps.ResultName != null) {
ResultOutput = new BinaryWriter (File.Create (searchOps.ResultName + ".tmp"));
}
int qid = 0;
long totaltime = 0;
SearchCost totalCost = new SearchCost (0, 0);
if (ResultOutput != null) {
var reslist = new ResultList (searchOps.IndexName, searchOps.QueryName);
// Dirty.SerializeBinary (Output, reslist);
reslist.Save (ResultOutput);
}
foreach (CommandQuery qItem in qReader) {
long tstart = DateTime.Now.Ticks;
SearchCost startCost = index.Cost;
IResult res;
var qobj = qItem.QObj;
if (qobj == null) {
qobj = index.DB.Parse (qItem.QRaw, true);
}
if (qItem.QTypeIsRange) {
res = index.SearchRange (qobj, qItem.QArg);
} else {
res = index.SearchKNN (qobj, (int)qItem.QArg);
}
var qraw = qItem.QRaw;
if (qraw.Length > 1024) {
qraw = "<very-large-qraw>";
}
SearchCost finalCost = index.Cost;
finalCost.Internal -= startCost.Internal;
finalCost.Total -= startCost.Total;
totalCost.Internal += finalCost.Internal;
totalCost.Total += finalCost.Total;
long time = DateTime.Now.Ticks - tstart;
totaltime += time;
ResultInfo info = new ResultInfo (qid, qItem.EncodeQTypeQArgInSign(), qraw, finalCost, new TimeSpan (time), res);
Console.WriteLine ("== qid: {0}", qid);
Console.WriteLine ("== index: {0}, db: {1}, result: {2}", index, index.DB.Name, searchOps.ResultName);
if (ResultOutput != null) {
// Dirty.SerializeBinary (ResultOutput, info);
info.Save (ResultOutput);
}
Console.WriteLine (info.ToString (searchOps.ShowMaxResult, null));
qid++;
}
if (ResultOutput != null) {
ResultOutput.Close ();
if (File.Exists (searchOps.ResultName)) {
File.Delete (searchOps.ResultName);
}
File.Move (searchOps.ResultName + ".tmp", searchOps.ResultName);
}
Console.WriteLine ("Number queries: {0}", qid);
Console.WriteLine ("Average total-numdists: {0}", (totalCost.Total + 0.0) / qid);
Console.WriteLine ("Average internal-distances: {0}", (totalCost.Internal + 0.0) / qid);
Console.WriteLine ("Average external-distances: {0}", (totalCost.Total - totalCost.Internal + 0.0) / qid);
Console.WriteLine ("Total search time: {0}", (new TimeSpan (totaltime)).TotalSeconds);
Console.WriteLine ("Average search time: {0}", (new TimeSpan (totaltime / qid)).TotalSeconds);
}
/// <summary>
///
/// </summary>
/// <param name="res">
/// A <see cref="ResultInfo"/>
/// </param>
public void Extend(ResultInfo res)
{
this.cost.External += res.cost.External;
this.cost.Internal += res.cost.Internal;
this.time = this.time.Add (res.time);
foreach (ResultPair p in res.result) {
this.result.Add (p);
}
}
/// <summary>
/// Queries for the cardinality of the unions
/// </summary>
public double Recall(ResultInfo basis, int basisSize, int resultSize)
{
if (resultSize == 0) {
return 0;
}
var intersection = this.IntersectCardinality (basis, basisSize, resultSize);
if (intersection == 0) {
return 0;
}
double r = intersection * 1.0 / basisSize;
if (r > 1) {
throw new ArgumentException (String.Format ("recall: {0}, basis: {1}, result: {2}", r, basisSize, resultSize));
}
return r;
}
/// <summary>
/// Cardinality of the unions
/// </summary>
public int UnionCardinality(ResultInfo n, int basisSize, int resultSize)
{
HashSet<int> ts = new HashSet<int> ();
HashSet<int> ns = new HashSet<int> ();
int i = 0;
foreach (ResultPair p in this.result) {
ts.Add (p.docid);
i++;
if (i >= resultSize) {
break;
}
}
i = 0;
foreach (ResultPair p in n.result) {
ns.Add (p.docid);
i++;
if (i >= basisSize) {
break;
}
}
ts.UnionWith (ns);
return ts.Count;
}
/// <summary>
/// Search shell (not interactive at this level)
/// </summary>
public static void Search(Index index, IEnumerable<CommandQuery> qReader, ShellSearchOptions searchOps, IEnumerable<string> args)
{
string names = null;
string[] dbnames = null;
if (searchOps.Names != null) {
dbnames = File.ReadAllLines (names);
}
BinaryWriter ResultOutput = null;
if (searchOps.ResultName != null) {
ResultOutput = new BinaryWriter (File.Create (searchOps.ResultName + ".tmp"));
}
SortedDictionary<double, int> avg_hist = new SortedDictionary<double, int> ();
int qid = 0;
long totaltime = 0;
SearchCost totalCost = new SearchCost (0, 0);
if (ResultOutput != null) {
var reslist = new ResultList (searchOps.IndexName, searchOps.QueryName);
// Dirty.SerializeBinary (Output, reslist);
reslist.Save (ResultOutput);
}
foreach (CommandQuery qItem in qReader) {
long tstart = DateTime.Now.Ticks;
SearchCost startCost = index.Cost;
IResult res;
if (qItem.QTypeIsRange) {
res = index.ParseSearch (qItem.QRaw, qItem.QArg);
} else {
res = index.ParseKNNSearch (qItem.QRaw, (int)qItem.QArg);
}
SearchCost finalCost = index.Cost;
finalCost.Internal -= startCost.Internal;
finalCost.External -= startCost.External;
totalCost.Internal += finalCost.Internal;
totalCost.External += finalCost.External;
long time = DateTime.Now.Ticks - tstart;
totaltime += time;
if (searchOps.Filter != null) {
res = searchOps.Filter (qItem.QRaw, qItem.QArg, res, index);
}
SortedDictionary<double, int> hist = new SortedDictionary<double, int> ();
if (searchOps.ShowHist) {
foreach (ResultPair p in res) {
if (hist.ContainsKey (p.dist)) {
hist [p.dist]++;
} else {
hist [p.dist] = 1;
}
}
foreach (var p in hist) {
if (avg_hist.ContainsKey (p.Key)) {
avg_hist [p.Key] += p.Value;
} else {
avg_hist [p.Key] = p.Value;
}
}
if (avg_hist.Count > 1000) {
searchOps.ShowHist = false;
Console.WriteLine ("WARNING: Histogram of distances was disabled because there are too many bins");
}
}
ResultInfo info = new ResultInfo (qid, qItem.EncodeQTypeQArgInSign(), qItem.QRaw, finalCost, new TimeSpan (time), res);
if (ResultOutput != null) {
// Dirty.SerializeBinary (ResultOutput, info);
info.Save (ResultOutput);
}
Console.WriteLine (info.ToString (searchOps.ShowMaxResult, dbnames));
if (searchOps.ShowHist) {
Console.WriteLine ("Distance histogram (dist => counter)");
foreach (KeyValuePair<double, int> xp in hist) {
Console.Write ("({0} => {1}), ", xp.Key, xp.Value);
}
Console.WriteLine ("<TheEnd>");
}
Console.WriteLine ("Number Results: {0}", res.Count);
qid++;
}
if (searchOps.ShowHist) {
Console.WriteLine ("Average Distance histogram (dist => counter)");
foreach (KeyValuePair<double, int> xp in avg_hist) {
Console.Write ("({0} => {1}), ", xp.Key, ((double)xp.Value) / qid);
}
Console.WriteLine ("<TheEnd>");
}
if (ResultOutput != null) {
ResultOutput.Close ();
if (File.Exists (searchOps.ResultName)) {
File.Delete (searchOps.ResultName);
}
File.Move (searchOps.ResultName + ".tmp", searchOps.ResultName);
}
Console.WriteLine ("Number queries: {0}", qid);
Console.WriteLine ("Average numdists: {0}", (totalCost.Internal + totalCost.External + 0.0) / qid);
Console.WriteLine ("Total search time: {0}", (new TimeSpan (totaltime)).TotalSeconds);
Console.WriteLine ("Average search time: {0}", (new TimeSpan (totaltime / qid)).TotalSeconds);
}